import os from typing import Tuple, List import argparse import re from pathlib import Path from pydub import AudioSegment import torch from TTS.api import TTS import requests import prompt OUT_DIR = 'out' SPEAKER_WAV = 'speaker.wav' LANG = "de" tts = None try: with open('apikey.secret') as f: api_key = f.read().strip() except FileNotFoundError: print('Couldn\'t read API key from file \'apikey.secret\'w. Does it exist?') def is_float(s: str) -> bool: try: float(s) return True except ValueError: return False def has_word_characters(s: str) -> bool: if re.search(r'\w', s): return True else: return False def transform_string(input_str: str) -> str: """ This method transforms strings like "Der Gauner, die Gauner" into der_gauner_die_gauner """ output_str = (input_str .lower() .replace(' ', '_') .replace(',', '') .replace(';', '') .replace('.', '') .replace('/', '') .replace('\\', '') .replace(']', '') .replace('[', '') .replace('(', '') .replace(')', '')) return output_str def get_tts_lazy() -> TTS: global tts if tts is None: device = "cuda" if torch.cuda.is_available() else "cpu" tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device) return tts def do_tts(text: str, speaker_wav: str, language: str, mp3_path: str, multisample: int = 4): """Generates audio file from text.""" os.makedirs('.tmp', exist_ok=True) # to prevent long extra phrases, run `multisample` times, and take shortest (smallest) sample for i in range(multisample): get_tts_lazy().tts_to_file(text=text, speaker_wav=speaker_wav, language=language, file_path=f".tmp/temp{i}.wav") shortest_sound = None # load from file and convert to mp3 for i in range(multisample): sound = AudioSegment.from_file(f".tmp/temp{i}.wav") if shortest_sound is None: shortest_sound = sound elif len(sound) < len(shortest_sound): shortest_sound = sound sound = shortest_sound sound = sound.set_frame_rate(44100) # Set frame rate to 44.1kHz, high quality sound = sound.set_sample_width(2) # 2 byte (16 bit) samples, high quality sound = sound.set_channels(2) # make it stereo # Export as high quality mp3 sound.export(mp3_path, format="mp3", bitrate="192k") # Export with high quality bitrate temp = "passieren; der Schaden, die Schäden; der Start, die Starts; die Strecke, die Strecken; der Verkehr; wenden; das Zeichen, die Zeichen; aussteigen; ausweichen; die Autobahn, die Autobahnen; der Bord, die Borde; die Brücke, die Brücken; einholen; einsteigen; entgegenkommen; fort; freigeben; der Hafen, die Häfen; der Halt, die Halte; die Kurve, die Kurven; laden; mobil; der Parkplatz, die Parkplätze; rollen; das Signal, die Signale; sperren; die Station, die Stationen; stoppen; das Tempo, die Tempos; das Ticket, die Tickets; der Transport, die Transporte; transportieren; der Tunnel, die Tunnel; der Unfall, die Unfälle; verkehren; verpassen" def extract_words_from_cards(cards: [Tuple[str, str]], temp: float = 0.0) -> List[str]: url = "https://api.perplexity.ai/chat/completions" left_sides = [card[0] for card in cards] query_words = ';'.join(left_sides) # german words payload = { "model": "mixtral-8x7b-instruct", # "model": "pplx-70b-chat", "messages": [ { "role": "system", "content": prompt.CARDS_TO_WORDS_PROMPT # prompt.CARDS_TO_WORDS_PROMPT # "content": prompt.LLAMA_CARDS_TO_WORDS_PROMPT }, { "role": "user", "content": "Here are Anki Cards to transform into speakable phrases: \n" + query_words + "\nMake sure to ONLY output a string of semicolon-separated speakable phrases. DO NOT write anything else!" } ], "temperature": temp, "presence_penalty": 0 } headers = { "accept": "application/json", "content-type": "application/json", "authorization": f"Bearer {api_key}", } while True: response = requests.post(url, json=payload, headers=headers) if response.status_code != 200: print("Perplexity API error") user_input = input("Retry? [Y/n] > ") if user_input.strip().lower() in ['n', 'no']: print("exiting...") exit(0) else: continue else: break print((response.json())) # split response at ;, and then return as list content = response.json()['choices'][0]['message']['content'] print(f'query: {query_words}') print(f'content: {content}') words = content.split(";") words = [word.split('\n')[0].strip() for word in words] # remove eventual newlines (e.g. comments) if len(words[-1]) == 0 or not has_word_characters(words[-1]): words = words[:-1] return words def process_text_to_audio(phrases_and_filenames: [(str, str)]): os.makedirs(OUT_DIR, exist_ok=True) for idx, phrase_and_filename in enumerate(phrases_and_filenames, 1): phrase, filename = phrase_and_filename mp3_path = os.path.join(OUT_DIR, f"{filename}.mp3") do_tts(phrase, 'speaker.wav', 'de', mp3_path) def main(): parser = argparse.ArgumentParser(description="Parse markdown note for Anki cards.") parser.add_argument('note_file', type=str, help='The path of the markdown note file.') parser.add_argument('--out-file', type=str, default='out.md', help='The output file.') parser.add_argument('--obsidian-format', action='store_true', help='Use Obsidian path format.') parser.add_argument("--batch-size", type=int, default=64, help='Number of cards sent to the LLM at one time, and processed in bulk. Default 64') parser.add_argument('--multisample', type=int, default=4, help='Number of audio generations per batch. Reduces audio with arbitrary sounds for short cards. Default 4') parser.add_argument('--multisample-multiply-limit', type=int, default=8, help='If a phrase is shorter than the multisample multiply limit, significantly more audio generations (generally *3) are done to improve quality. Set to 0 to disable. Default 8') parser.add_argument('--multisample-multiply', type=int, default=3, help='Sets the multiplier for additional audio generations when a phrase is shorter than the multisample multiply limit. Default is 3') args = parser.parse_args() # Check if the note file exists if not Path(args.note_file).exists(): raise FileNotFoundError(f"File {args.note_file} does not exist.") # Read file and parse for Anki cards with open(args.note_file, 'r') as f: content = f.read() matches = re.findall(r"^Q: (.+)\nA: (.+)\n", content, re.MULTILINE) # truncate for debug # matches = matches[:129] batch_size = args.batch_size # process in blocks of batch_size out_content = content for i in range(len(matches) // batch_size + 1): to_match = matches[i * batch_size:i * batch_size + batch_size] correct_words_generated = False words = None subbatch_size = batch_size cur_temp = 0 # process cards while not correct_words_generated: batches = [to_match[i:i + subbatch_size] for i in range(0, len(to_match), subbatch_size)] words = [] for batch in batches: words += extract_words_from_cards(batch, temp=cur_temp) if len(words) != len(to_match): print(f'generated words len ({len(words)}) != matches len ({len(to_match)})') print( f'Current Batch Size is {subbatch_size}, temp is {cur_temp}. If this happens repeatedly, try reducing the batch size.') userinput = input('Try again? [Y/n/split/temp] > ') if userinput.strip().lower() in ['n', 'no']: print("aborting...") exit(0) elif userinput.strip().lower() in ['s', 'split']: subbatch_size = max(1, subbatch_size // 2) # generate batches elif userinput.strip().lower() in ['t', 'temp']: while True: new_temp = input(f"cur temp: {cur_temp}. Input new temp (0-2) > ") if is_float(new_temp) and 0 <= float(new_temp) <= 2: cur_temp = float(new_temp) break else: print("Must be numeric (float) between 0 and 2") else: print("trying again...") else: correct_words_generated = True # assert len(words) == len(to_match), f'generated words len ({len(words)}) != matches len ({len(to_match)})' print(str(words)) filenames = [] for j, word in enumerate(words): print(f'speaker-ifying word {word} ({j + i * batch_size} of {len(matches)})') word = word.strip() filename = f"{transform_string(word)}.mp3" filenames.append(filename) multisample = args.multisample if len(word) < args.multisample_multiply_limit: multisample *= args.multisample_multiply # generate 3x as many for short words / phrases do_tts(word, SPEAKER_WAV, LANG, f'./{OUT_DIR}/{filename}', multisample=multisample) for i, (question, answer) in enumerate(to_match): out_content = out_content.replace(f"Q: {question}", f"Q: {question} {'![[' if args.obsidian_format else f'![](./{OUT_DIR}/'}{filenames[i]}{']]' if args.obsidian_format else ')'}") # open out file for writing out_file = args.out_file with open(out_file, 'w') as f: f.write(out_content) if __name__ == '__main__': main()