INITIAL COMMIT
This commit is contained in:
commit
9121ad819d
12
.fleet/run.json
Normal file
12
.fleet/run.json
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
"configurations": [
|
||||
{
|
||||
"type": "command",
|
||||
"name": "Command configuration",
|
||||
"program": "poetry",
|
||||
"args": ["run", "python", "main.py"],
|
||||
},
|
||||
|
||||
|
||||
]
|
||||
}
|
||||
23
README.md
Normal file
23
README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Flüsterpost
|
||||
|
||||
Flüsterpost takes audio files and turns them into text. It's made to do so easily, quickly, and locally.
|
||||
|
||||
## How to Run
|
||||
|
||||
This repo uses the poetry package manager for Python. If you don't have it, you'll need to install it before using this.
|
||||
(i can recommend it. it's great.)
|
||||
|
||||
`cd` into the repository. Then, run `poetry install`.
|
||||
Then, do `poetry run main.py`.
|
||||
|
||||
### Deps
|
||||
This project uses [flet](https://flet.dev/) for the GUI. On Linux systems, you'll need `gstreamer` installed.
|
||||
You probably have this shipped with your system, but if you get the error message
|
||||
`error while loading shared libraries: libgstapp-1.0.so.0: cannot open shared object file: No such file or directory`,
|
||||
install it. More info on the [flet docs page](https://flet.dev/docs/guides/python/getting-started).
|
||||
|
||||
## License
|
||||
This project is licensed under the MIT license.
|
||||
|
||||
## Contributions
|
||||
Any contributions to this project will be licensed under the MIT license, unless explicitly noted otherwise.
|
||||
BIN
__pycache__/nn_model_manager.cpython-311.pyc
Normal file
BIN
__pycache__/nn_model_manager.cpython-311.pyc
Normal file
Binary file not shown.
BIN
__pycache__/utils.cpython-311.pyc
Normal file
BIN
__pycache__/utils.cpython-311.pyc
Normal file
Binary file not shown.
322
main.py
Normal file
322
main.py
Normal file
@ -0,0 +1,322 @@
|
||||
import os
|
||||
import pprint
|
||||
import traceback
|
||||
|
||||
import utils
|
||||
import flet as ft
|
||||
|
||||
from typing import DefaultDict
|
||||
|
||||
import pygame
|
||||
|
||||
import nn_model_manager as mm
|
||||
|
||||
|
||||
def main(page):
|
||||
pygame.mixer.init()
|
||||
|
||||
first_name = ft.Ref[ft.TextField]()
|
||||
last_name = ft.Ref[ft.TextField]()
|
||||
greetings = ft.Ref[ft.Column]()
|
||||
|
||||
file_tree = ft.Ref[ft.Column]()
|
||||
file_tree_empty_text = ft.Ref[ft.Text]()
|
||||
|
||||
load_model_text = ft.Ref[ft.Text]()
|
||||
model_size_select = ft.Ref[ft.Dropdown]()
|
||||
model_device_select = ft.Ref[ft.Dropdown]()
|
||||
# model_bits_select = ft.Ref[ft.Dropdown]()
|
||||
model_load_unload_button = ft.Ref[ft.IconButton]()
|
||||
model_loading_spinner = ft.Ref[ft.ProgressRing]()
|
||||
|
||||
transcribe_buttons: list[ft.Ref[ft.IconButton]] = []
|
||||
|
||||
output_text_container = ft.Ref[ft.Container]()
|
||||
output_text_col = ft.Ref[ft.Column]()
|
||||
|
||||
def generate_file_tree(path: str, tree_dict: dict | DefaultDict):
|
||||
if path[-1] == os.sep:
|
||||
path = path[:-1]
|
||||
|
||||
folder_name = utils.get_last_segment(path)
|
||||
print(f"DEBUG: generating tree for folder {folder_name}")
|
||||
|
||||
# find folders, and add dict for each
|
||||
print(f"adding name {folder_name} to ui")
|
||||
|
||||
controls = [
|
||||
ft.Row(
|
||||
[
|
||||
ft.Icon(ft.icons.FOLDER, color=ft.colors.BLUE),
|
||||
ft.Text(folder_name, size=14, weight=ft.FontWeight.BOLD),
|
||||
]
|
||||
)
|
||||
]
|
||||
|
||||
for folder_name, value in tree_dict.items():
|
||||
if folder_name == utils.FILES_KEY or folder_name == '.':
|
||||
continue # skip for now
|
||||
|
||||
controls.append(generate_file_tree(path + os.sep + folder_name, value))
|
||||
|
||||
# now folders are there, let's do files
|
||||
if utils.FILES_KEY not in tree_dict and '.' in tree_dict:
|
||||
tree_dict = tree_dict['.'] # if root dir, enter root dir (.) directory
|
||||
|
||||
files_controls = []
|
||||
|
||||
for file in tree_dict[utils.FILES_KEY]:
|
||||
control = [ft.Text(file)]
|
||||
|
||||
if not file.endswith('.mp3'):
|
||||
continue
|
||||
|
||||
def start_playing(filepath: str, button_ref: ft.Ref[ft.IconButton]):
|
||||
print(f"trying to play {filepath}...")
|
||||
if pygame.mixer.music.get_busy() or not os.path.isfile(filepath):
|
||||
return
|
||||
|
||||
print("starting playback")
|
||||
|
||||
pygame.mixer.music.load(filepath)
|
||||
pygame.mixer.music.play()
|
||||
|
||||
button_ref.current.icon = ft.icons.PAUSE_CIRCLE_FILLED_OUTLINED
|
||||
button_ref.current.on_click = lambda _, f=filepath, r=button_ref: stop_playing(f, r)
|
||||
page.update()
|
||||
|
||||
def stop_playing(filepath: str, button_ref: ft.Ref[ft.IconButton]):
|
||||
print("stopping playback")
|
||||
|
||||
pygame.mixer.music.stop()
|
||||
|
||||
button_ref.current.icon = ft.icons.PLAY_CIRCLE_OUTLINED
|
||||
button_ref.current.on_click = lambda _, f=filepath, r=button_ref: start_playing(f, r)
|
||||
page.update()
|
||||
|
||||
full_file_path = path + os.sep + file
|
||||
|
||||
_button_ref = ft.Ref[ft.IconButton]()
|
||||
|
||||
control.append(ft.IconButton(icon=ft.icons.PLAY_CIRCLE_OUTLINED, ref=_button_ref,
|
||||
on_click=lambda _, f=full_file_path, r=_button_ref: start_playing(f, r)))
|
||||
|
||||
def transcribe(filepath: str):
|
||||
print(f"DEBUG: trying to transcribe file {filepath}")
|
||||
if not mm.is_model_loaded() or not filepath.endswith('.mp3'):
|
||||
return
|
||||
|
||||
print(f"DEBUG: starting transcription")
|
||||
output_text_container.current.alignment = ft.alignment.center
|
||||
output_text_col.current.controls = [ft.ProgressRing()]
|
||||
|
||||
# set all transcribe buttons to disabled
|
||||
for btn in transcribe_buttons:
|
||||
btn.current.disabled = True
|
||||
page.update()
|
||||
|
||||
try:
|
||||
segments, info = mm.transcribe_from_file(filepath)
|
||||
|
||||
txt = ''
|
||||
|
||||
for seg in segments:
|
||||
txt += seg.text + '\n'
|
||||
|
||||
output_text_container.current.alignment = ft.alignment.top_left
|
||||
output_text_col.current.controls = [ft.Text(txt, selectable=True)] # TODO
|
||||
|
||||
except Exception as e:
|
||||
output_text_container.current.alignment = ft.alignment.center
|
||||
output_text_col.current.controls = [ft.Text(f"Transcribing failed: {str(e)}")] # TODO
|
||||
|
||||
finally:
|
||||
# set all transcribe buttons to disabled
|
||||
for btn in transcribe_buttons:
|
||||
btn.current.disabled = False
|
||||
page.update()
|
||||
|
||||
|
||||
transcribe_button_ref = ft.Ref[ft.IconButton]()
|
||||
|
||||
control.append(ft.IconButton(icon=ft.icons.FORMAT_ALIGN_LEFT, disabled=not mm.is_model_loaded(), ref=transcribe_button_ref,
|
||||
on_click=lambda _, f=full_file_path: transcribe(f)))
|
||||
|
||||
transcribe_buttons.append(transcribe_button_ref)
|
||||
|
||||
files_controls.append(ft.Row(control))
|
||||
|
||||
if len(files_controls) == 0:
|
||||
files_controls.append(ft.Text('No mp3 Files found', color='grey'))
|
||||
|
||||
return ft.Row([
|
||||
ft.VerticalDivider(),
|
||||
ft.Column(controls + [ft.Row([ft.VerticalDivider(), ft.Column(files_controls)])])
|
||||
]
|
||||
)
|
||||
|
||||
def btn_click(e):
|
||||
greetings.current.controls.append(
|
||||
ft.Text(f"Hello, {first_name.current.value} {last_name.current.value}!")
|
||||
)
|
||||
first_name.current.value = ""
|
||||
last_name.current.value = ""
|
||||
page.update()
|
||||
first_name.current.focus()
|
||||
|
||||
def on_dialog_result(e: ft.FilePickerResultEvent):
|
||||
path = e.path
|
||||
if path:
|
||||
print(f"path is {path}")
|
||||
try:
|
||||
if os.path.isdir(path):
|
||||
tree = utils.build_file_tree(path)
|
||||
|
||||
if '.' in tree: # if there is actually a proper file tree
|
||||
# add to view
|
||||
file_tree.current.controls.append(
|
||||
generate_file_tree(path, utils.defaultdict_to_dict(tree))
|
||||
)
|
||||
file_tree_empty_text.current.visible = False
|
||||
|
||||
page.update()
|
||||
except e:
|
||||
print("didn't work aaa") # TODO: fix
|
||||
|
||||
def load_model():
|
||||
|
||||
load_model_text.current.value = 'Loading... This may take a while.'
|
||||
|
||||
model_size_select.current.disabled = True
|
||||
model_device_select.current.disabled = True
|
||||
# model_bits_select.current.disabled = True
|
||||
model_load_unload_button.current.disabled = True
|
||||
model_loading_spinner.current.visible = True
|
||||
page.update()
|
||||
|
||||
try:
|
||||
mm.set_model(
|
||||
size=model_size_select.current.value or 'base',
|
||||
device=model_device_select.current.value or 'auto',
|
||||
# compute_type=model_bits_select.current.value or '16bit',
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"loading model failed. Exception: {str(e)}")
|
||||
print(traceback.format_exc())
|
||||
load_model_text.current.value = f'Loading failed. Reason:\n{str(e)}'
|
||||
model_size_select.current.disabled = False
|
||||
model_device_select.current.disabled = False
|
||||
# model_bits_select.current.disabled = False
|
||||
|
||||
# raise e
|
||||
|
||||
model_loading_spinner.current.visible = False
|
||||
model_load_unload_button.current.disabled = False
|
||||
|
||||
if mm.is_model_loaded():
|
||||
load_model_text.current.value = f'Loaded.'
|
||||
model_load_unload_button.current.icon = ft.icons.CLOSE
|
||||
model_load_unload_button.current.on_click = lambda _: unload_model()
|
||||
|
||||
# if successful, save to shared preferences
|
||||
page.client_storage.set('model_size', model_size_select.current.value)
|
||||
page.client_storage.set('device_select', model_device_select.current.value)
|
||||
|
||||
# set all transcribe buttons to enabled
|
||||
for btn in transcribe_buttons:
|
||||
btn.current.disabled = False
|
||||
|
||||
page.update()
|
||||
|
||||
def unload_model():
|
||||
model_load_unload_button.current.disabled = True
|
||||
|
||||
# set all transcribe buttons to disabled
|
||||
for btn in transcribe_buttons:
|
||||
btn.current.disabled = True
|
||||
|
||||
page.update()
|
||||
|
||||
if mm.is_model_loaded():
|
||||
mm.unload_model()
|
||||
|
||||
load_model_text.current.value = 'Select parameters, and then load transcription model.'
|
||||
model_size_select.current.disabled = False
|
||||
model_device_select.current.disabled = False
|
||||
# model_bits_select.current.disabled = False
|
||||
model_load_unload_button.current.disabled = False
|
||||
model_load_unload_button.current.icon = ft.icons.START
|
||||
model_load_unload_button.current.on_click = lambda _: load_model()
|
||||
model_loading_spinner.current.visible = False
|
||||
page.update()
|
||||
|
||||
|
||||
|
||||
|
||||
# set up file picker
|
||||
file_picker = ft.FilePicker(on_result=on_dialog_result)
|
||||
|
||||
page.overlay.append(file_picker)
|
||||
|
||||
page.add(
|
||||
ft.Text("Flüsterpost", style=ft.TextThemeStyle.TITLE_LARGE),
|
||||
ft.Divider()
|
||||
)
|
||||
|
||||
page.add(
|
||||
ft.ResponsiveRow([
|
||||
ft.Container(
|
||||
ft.Column([
|
||||
ft.ElevatedButton("Add Folder", on_click=lambda _: file_picker.get_directory_path()),
|
||||
ft.Column(ref=file_tree, scroll=ft.ScrollMode.ALWAYS, expand=True),
|
||||
# ft.ListView(ref=file_tree),
|
||||
ft.Text("No Folder Open Yet", style=ft.TextTheme.body_small, color="grey",
|
||||
ref=file_tree_empty_text),
|
||||
], expand=True), expand=True, col=4),
|
||||
ft.Container(expand=True, content=ft.Column(expand=True, controls=[
|
||||
ft.Column([
|
||||
ft.Text('Select parameters, and then load transcription model.', ref=load_model_text),
|
||||
ft.Row([
|
||||
ft.Dropdown(
|
||||
ref=model_size_select,
|
||||
width=100,
|
||||
hint_text='model size',
|
||||
value=page.client_storage.get('model_size') if page.client_storage.contains_key('model_size') else 'base',
|
||||
options=[ft.dropdown.Option(x) for x in mm.ModelSize.__args__], # __args__ is not perfect here. But works.
|
||||
),
|
||||
ft.Dropdown(
|
||||
ref=model_device_select,
|
||||
width=100,
|
||||
hint_text='device',
|
||||
value=page.client_storage.get('device_select') if page.client_storage.contains_key('device_select') else 'auto',
|
||||
options=[ft.dropdown.Option(x) for x in mm.Device.__args__] # __args__ is not perfect here. But works.
|
||||
),
|
||||
# ft.Dropdown(
|
||||
# ref=model_bits_select,
|
||||
# width=100,
|
||||
# hint_text='bits',
|
||||
# value='16bit',
|
||||
# options=[ft.dropdown.Option(x) for x in mm.ComputeType.__args__] # __args__ is not perfect here. But works.
|
||||
#),
|
||||
ft.IconButton(
|
||||
icon=ft.icons.START,
|
||||
ref=model_load_unload_button,
|
||||
on_click=lambda _: load_model(),
|
||||
),
|
||||
ft.ProgressRing(ref=model_loading_spinner, visible=False)
|
||||
])
|
||||
]),
|
||||
ft.Container(expand=True, padding=12, border=ft.border.all(2, 'grey'),
|
||||
alignment=ft.alignment.center,
|
||||
ref=output_text_container,
|
||||
content=ft.Column(
|
||||
[ft.Text('Nothing to see here!', text_align=ft.TextAlign.CENTER)],
|
||||
ref=output_text_col,
|
||||
expand=True,
|
||||
scroll=ft.ScrollMode.ADAPTIVE)),
|
||||
]), col=8)
|
||||
], expand=True),
|
||||
)
|
||||
|
||||
|
||||
ft.app(target=main)
|
||||
65
nn_model_manager.py
Normal file
65
nn_model_manager.py
Normal file
@ -0,0 +1,65 @@
|
||||
import threading
|
||||
|
||||
from faster_whisper import WhisperModel
|
||||
import faster_whisper
|
||||
from typing import Literal, Iterable, Tuple
|
||||
|
||||
_model: WhisperModel | None = None
|
||||
|
||||
ModelSize = Literal["tiny", "base", "small", "medium", "large-v1", "large-v2"]
|
||||
Device = Literal["cuda", "cpu", "auto"]
|
||||
ComputeType = Literal["8bit", "16bit", "32bit"]
|
||||
|
||||
|
||||
def set_model(size: ModelSize, device: Device): #, compute_type: ComputeType):
|
||||
|
||||
'''
|
||||
compute = None
|
||||
if compute_type == '8bit':
|
||||
if device == 'cuda' or device == 'auto':
|
||||
compute = 'int8_float16'
|
||||
else:
|
||||
compute = 'int8'
|
||||
elif compute_type == '16bit':
|
||||
if device == 'cuda' or device == 'auto':
|
||||
compute = 'int8'
|
||||
else:
|
||||
raise Exception("Cannot do 16 bit computing on CPU")
|
||||
elif compute_type == '32bit':
|
||||
compute = 'float'
|
||||
else:
|
||||
raise Exception(f"Invalid Compute / Device configuration (device {device} with {compute_type})")
|
||||
'''
|
||||
|
||||
global _model
|
||||
_model = WhisperModel(size, device=device)
|
||||
|
||||
|
||||
def unload_model():
|
||||
if not is_model_loaded():
|
||||
return
|
||||
|
||||
global _model
|
||||
_model = None # TODO: check if this works
|
||||
|
||||
def is_model_loaded() -> bool:
|
||||
global _model
|
||||
return _model is not None
|
||||
|
||||
|
||||
def transcribe_from_file(mp3_path: str) -> Tuple[Iterable[faster_whisper.transcribe.Segment], faster_whisper.transcribe.TranscriptionInfo] | None:
|
||||
"""
|
||||
Transcribe audio from an MP3 file.
|
||||
Note that this can - and will - crash if you don't catch exceptions.
|
||||
|
||||
If the model isn't loaded yet, this will return None.
|
||||
Otherwise, it will return the raw transcription from `faster-whisper`.
|
||||
"""
|
||||
if not is_model_loaded():
|
||||
return None
|
||||
|
||||
global _model
|
||||
segments, info = _model.transcribe(mp3_path, beam_size=5)
|
||||
# transcribe, and throw all exceptions to application to handle
|
||||
|
||||
return segments, info
|
||||
2043
poetry.lock
generated
Normal file
2043
poetry.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
19
pyproject.toml
Normal file
19
pyproject.toml
Normal file
@ -0,0 +1,19 @@
|
||||
[tool.poetry]
|
||||
name = "fluesterpost"
|
||||
version = "0.1.0"
|
||||
description = "App for transcribing audio files to text"
|
||||
authors = ["Yandrik <me@yandrik.dev>"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.11"
|
||||
flet = "^0.10.3"
|
||||
faster-whisper = "^0.9.0"
|
||||
pygame = "^2.5.2"
|
||||
torch = "2.0.0"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
285
temp
Normal file
285
temp
Normal file
@ -0,0 +1,285 @@
|
||||
{ '.': { '_i_files': [ 'getting-started.md',
|
||||
'inbox.md',
|
||||
'readme.md',
|
||||
'skill_stack.md',
|
||||
'player_famillarity.md',
|
||||
'todo.md']},
|
||||
'.foam': { '_i_files': [],
|
||||
'templates': { '_i_files': [ 'anki-cards.md',
|
||||
'your-first-template.md',
|
||||
'zettel-refs.md',
|
||||
'zettel.md']}},
|
||||
'.git': { '_i_files': [ 'description',
|
||||
'packed-refs',
|
||||
'HEAD',
|
||||
'config',
|
||||
'COMMIT_EDITMSG',
|
||||
'FETCH_HEAD',
|
||||
'ORIG_HEAD',
|
||||
'index'],
|
||||
'branches': {'_i_files': []},
|
||||
'hooks': { '_i_files': [ 'applypatch-msg.sample',
|
||||
'commit-msg.sample',
|
||||
'post-update.sample',
|
||||
'pre-applypatch.sample',
|
||||
'pre-commit.sample',
|
||||
'pre-merge-commit.sample',
|
||||
'pre-push.sample',
|
||||
'pre-receive.sample',
|
||||
'push-to-checkout.sample',
|
||||
'update.sample',
|
||||
'fsmonitor-watchman.sample',
|
||||
'pre-rebase.sample',
|
||||
'prepare-commit-msg.sample']},
|
||||
'info': {'_i_files': ['exclude']},
|
||||
'logs': { '_i_files': ['HEAD'],
|
||||
'refs': { '_i_files': [],
|
||||
'heads': {'_i_files': ['master']},
|
||||
'remotes': { '_i_files': [],
|
||||
'origin': { '_i_files': [ 'HEAD',
|
||||
'master']}}}},
|
||||
'objects': { '01': { '_i_files': [ '92178cee01a3ad18cb621cac1ba64ad8e4e8a3']},
|
||||
'02': { '_i_files': [ '8c8217c849a00a010fec6cfa99a9e92e8c9375',
|
||||
'557a1b00b8c95a84b967c50c1e1aa7974a69ce']},
|
||||
'05': { '_i_files': [ 'f4816f07fc4f96c0c2010cf0d7449f00fcce19',
|
||||
'0a51af521fb44d8aacd7604d14b8766238d66f']},
|
||||
'0b': { '_i_files': [ 'f3fdf9465baf34897ad3252be2341bc1bf7db8']},
|
||||
'0f': { '_i_files': [ 'd094c8d57913f667e462632d49b2f052bab5de']},
|
||||
'11': { '_i_files': [ 'ae42db7bb29c0a5f1987e75db0c49fbdf8f957']},
|
||||
'13': { '_i_files': [ 'b1d98f14fd2acff4afa59cf80bf7150a44d8d5']},
|
||||
'18': { '_i_files': [ 'b5e3934b5d99910733c118c634fcd17c58b369']},
|
||||
'1d': { '_i_files': [ '8661f5e25c86324e1bba1933948c3900126c06']},
|
||||
'20': { '_i_files': [ '0e31f1ae6fcde97e64e5dc1ea7d5f10008dea4']},
|
||||
'24': { '_i_files': [ '5d1d3c006b36d48b007298349f2f33bf3b14d8']},
|
||||
'28': { '_i_files': [ 'b6a59bbb64bd2705e586f53a880ce2d4c00515']},
|
||||
'2b': { '_i_files': [ '25ddb4990745c5df76289a21a4d7d6e333d054']},
|
||||
'30': { '_i_files': [ 'e393907000c0219d86dbffd28bb1e548c7a078']},
|
||||
'32': { '_i_files': [ '711ab584d672d6e5aa62da57b5a45c8d231689']},
|
||||
'34': { '_i_files': [ '7998770c00191c4a76a470c5a4a1311c2265fc']},
|
||||
'35': { '_i_files': [ 'bb5e4cb76375d51162c6143c3ebd998f469714']},
|
||||
'38': { '_i_files': [ 'b873fbb24e1d38a5896f9161b0d4e98aec472f']},
|
||||
'39': { '_i_files': [ 'cdb1347792036be3a4ceb5fe49faace3c5f579']},
|
||||
'3a': { '_i_files': [ 'ba671d641c06e76036016b91e577f91a4b02ad',
|
||||
'44ad749af1d6dc5866356beb964f766eee5128']},
|
||||
'42': { '_i_files': [ 'a79846fbcee6a7b10a0ce6a6fca8994fc80876']},
|
||||
'45': { '_i_files': [ 'dc2d782c3fe9faf18d6517f11f183cf78f177a']},
|
||||
'53': { '_i_files': [ '196f6105b690fd51e81484deb1321976b1daa9']},
|
||||
'56': { '_i_files': [ '729062c3620d0cc078c7b7033f93e64982bfc4',
|
||||
'b44a575108bdb87f385b47c30f68d418a3eaaa']},
|
||||
'5c': { '_i_files': [ '13490222138a9d52e50f4782969284fd43e8ce']},
|
||||
'5f': { '_i_files': [ '3ff320973f57bb17f5abd596af5f73b15bd601']},
|
||||
'67': { '_i_files': [ 'e12b1d2f0a5307acc8f06fa0edb883ef86acfe']},
|
||||
'6e': { '_i_files': [ 'd2d0031d1ffb6c25f58046a1971661e0cf1cc4']},
|
||||
'70': { '_i_files': [ '92cf531a29dd7fdc0a4d09acb194c4a57d8803',
|
||||
'b9d0a5d2154b5be599f50ca39883642ef681ec']},
|
||||
'71': { '_i_files': [ '45b359be3dd773a9ccbc6aaeba7b9be9ba074c']},
|
||||
'74': { '_i_files': [ '83a25779adf27f8c37221076cb0e61a2025a5c']},
|
||||
'75': { '_i_files': [ '490a494632a140b2119a894e6521161d302769']},
|
||||
'76': { '_i_files': [ 'd9e26bda3f744c90704e94004f2350a3b145ba']},
|
||||
'78': { '_i_files': [ '4442d08c18a84bd0570ecc56518073f4a02430']},
|
||||
'79': { '_i_files': [ 'bcbd1fa1badcf3677e52ddd76721bf7ddfc115']},
|
||||
'7a': { '_i_files': [ '5177de86b242d198edb87bd20a5613f106e596']},
|
||||
'82': { '_i_files': [ '65b4da954fcdc135ffaf0019b31cdb228e59ec',
|
||||
'a06086a94bc1fa94b8f054e7c48c3a8d45a580']},
|
||||
'85': { '_i_files': [ '29e2398baf641413f194a652305c3632c757c7']},
|
||||
'88': { '_i_files': [ '83fca5338d4419f32e57377106986fea85f81e']},
|
||||
'8c': { '_i_files': [ '972d8832336104982d5f66037246ed75050c94']},
|
||||
'8d': { '_i_files': [ '200ac5a1dd8d235bb786bcdf94527297c6019c']},
|
||||
'90': { '_i_files': [ 'f26b565b09edc34d98bfe9b01dfc88d01b3e05']},
|
||||
'92': { '_i_files': [ '048c50fec01e3a6b9cc7d301b48f62f0d47ad6',
|
||||
'b464c0c03c745287234e4ccbbd0dc5285fc545']},
|
||||
'93': { '_i_files': [ 'a00093b5319ab79d4b7a9c490cb4e2a661907e']},
|
||||
'94': { '_i_files': [ '05bfdc224d3e5dfe9fe6116a4fd2955ab10a72']},
|
||||
'95': { '_i_files': [ '0027f29e86396e4d7416487d16100fc559fb2d',
|
||||
'7a01dc4f30e7f3424d004a17d7b5571d0e3879']},
|
||||
'96': { '_i_files': [ '2e3e24df9db27f23050dc22c057793b40455cc']},
|
||||
'97': { '_i_files': [ 'f9215e573ba5b7d2273bc7b368d025b37f0f7a']},
|
||||
'9c': { '_i_files': [ '6b61bb1fdfba8f5b242f7f98284a528a9e17c8']},
|
||||
'9d': { '_i_files': [ '9fe831567fe46ae777733412cad5157982c44a']},
|
||||
'9e': { '_i_files': [ '26dfeeb6e641a33dae4961196235bdb965b21b',
|
||||
'a7cd9ba86b57ada4037f24526eee3579efd26d']},
|
||||
'_i_files': [],
|
||||
'a6': { '_i_files': [ 'ffab47a1ec45c43ed1927ddc6097a088fcad4f']},
|
||||
'a8': { '_i_files': [ 'a84552bead93885c351952ca9781d4ddbe5780',
|
||||
'edeadbf5b40ab9eb7598d729263e270ddd9e63']},
|
||||
'b3': { '_i_files': [ '2e1b1d3d1bbd905aca74818820b184001fea13']},
|
||||
'b5': { '_i_files': [ 'cd6cf2d939352ff74ac7ca30018decc0647b65']},
|
||||
'b9': { '_i_files': [ '0227ebf336c51d2364f2c6ed9cb9896a236f49']},
|
||||
'ba': { '_i_files': [ '4fad78807ee9ec608eca8c6355b54146b52a68']},
|
||||
'bb': { '_i_files': [ '90408b3c6db106c5fe19e6bd142d2bf6b7b528']},
|
||||
'bc': { '_i_files': [ '4057206a45a261e46f8b6f4896d7c5b843876f']},
|
||||
'c0': { '_i_files': [ 'd770dce01d3d0ca290ddd7e494714960c77326',
|
||||
'190d5c75633f5ea107de72845dc5387a518400',
|
||||
'8336e9a181a2788e9bbcba484415698701638a',
|
||||
'0153d83030dad37d32e38628fe10e304f3377c']},
|
||||
'c3': { '_i_files': [ '31566dd29ec2f17a6ba736a2fe0a45fa65e89d']},
|
||||
'c4': { '_i_files': [ '1c5acd753957dda0d5bdeb25fb47e793080453']},
|
||||
'c8': { '_i_files': [ 'c365d89b8b1bad158cba37051fc32cbc784c65']},
|
||||
'ce': { '_i_files': [ '8bbb33980fd7c40f017d1063950fd8fdab2d6b']},
|
||||
'cf': { '_i_files': [ 'dc0b4420769cfc9219fa741447ae43b8129b75']},
|
||||
'd8': { '_i_files': [ 'ee8b9b4c05ce67098abbdfe9e351d63b21f18e',
|
||||
'eeb04612a7818d4c0872b312f4e2241f996190']},
|
||||
'd9': { '_i_files': [ '54566657c342600667bb4836f83a8407e32807']},
|
||||
'dd': { '_i_files': [ '2a192a1c242d0691fb8db2fdd08733c63a0dfb']},
|
||||
'e1': { '_i_files': [ '2b226b929c2c1996829b1dd45539f71fba1d4b']},
|
||||
'e5': { '_i_files': [ 'c523af28c3a765a63e7e5fbf44a40a44cd0d05']},
|
||||
'e6': { '_i_files': [ '80ac775c8245442f5dec43644a2aa7aac6dbc2',
|
||||
'a347474ea106d1b0979e5aead00e695710eb34']},
|
||||
'ea': { '_i_files': [ '514bc29f92feccb4fc8fb53dcbee24cb9863b2']},
|
||||
'ee': { '_i_files': [ '41eb2f83cb8a36fb5773cc3501a73d6b5a484f']},
|
||||
'ef': { '_i_files': [ '998d5afc023c58a624a7b01261f15b416ef286']},
|
||||
'f1': { '_i_files': [ '3dba7d7450e0ea935bc15b087c2e62e3f9f62b',
|
||||
'053bfb26234ebea971e874a0150b221f5f598d',
|
||||
'ae4aac4535bfe3d569f4376e1992acdd5e5fde']},
|
||||
'f2': { '_i_files': [ '6065a6e08f05d5d0dad0e8f9de825648938a04',
|
||||
'432313c1f432e922ed0dc4baea17c8d5043cfb']},
|
||||
'f3': { '_i_files': [ 'ebbe30d46a1d7d56b95ab0d64f64b7cfede59a']},
|
||||
'f6': { '_i_files': [ 'd65179b69864c0b1d88f4fa32b35a9089bfa83']},
|
||||
'f8': { '_i_files': [ '5851a1afaca442587934e62f1b182f3e09e3d1',
|
||||
'af88bf54d71d06fba7335ff4662722019a1f9b',
|
||||
'61ad4ac94746c2347df1aa6ae2dafafe7f9538']},
|
||||
'fb': { '_i_files': [ '8be1fd027914d656729cc5217b6f9a7c1cb4f3']},
|
||||
'fd': { '_i_files': [ '0d011ded41de68be2f0039c97890ed4117f7ff',
|
||||
'204664448169af25c5126c3c8c11917ec5cb6f']},
|
||||
'info': {'_i_files': []},
|
||||
'pack': { '_i_files': [ 'pack-8b0ef94d8431e1b37110e10fedb27318a852f391.pack',
|
||||
'pack-8b0ef94d8431e1b37110e10fedb27318a852f391.idx']}},
|
||||
'refs': { '_i_files': [],
|
||||
'heads': {'_i_files': ['master']},
|
||||
'remotes': { '_i_files': [],
|
||||
'origin': { '_i_files': [ 'HEAD',
|
||||
'master']}},
|
||||
'tags': {'_i_files': []}}},
|
||||
'.obsidian': { '_i_files': [ 'hotkeys.json',
|
||||
'core-plugins.json',
|
||||
'core-plugins-migration.json',
|
||||
'app.json',
|
||||
'appearance.json',
|
||||
'workspace.json',
|
||||
'graph.json',
|
||||
'community-plugins.json'],
|
||||
'plugins': { '_i_files': [],
|
||||
'dataview': { '_i_files': [ 'manifest.json',
|
||||
'main.js',
|
||||
'styles.css']},
|
||||
'obsidian-git': { '_i_files': [ 'manifest.json',
|
||||
'main.js',
|
||||
'styles.css',
|
||||
'data.json']}}},
|
||||
'.vscode': { '_i_files': [ 'extensions.json',
|
||||
'foam.json',
|
||||
'settings.json',
|
||||
'spellright.dict']},
|
||||
'_layouts': {'_i_files': ['home.html', 'page.html']},
|
||||
'abawo': { '_i_files': ['corporate-design.md'],
|
||||
'keant': { '_i_files': [],
|
||||
'emc': {'_i_files': ['keant-emc-testing.md']}}},
|
||||
'ai': { '_i_files': [ 'clustering.md',
|
||||
'gradient_descent.md',
|
||||
'nn_neurons.md'],
|
||||
'basics': { '_i_files': [ 'embedding.md',
|
||||
'latent-space.md',
|
||||
'few-shot.md',
|
||||
'zero-shot.md']},
|
||||
'combination': {'_i_files': ['query-net.md']},
|
||||
'ffn': {'_i_files': ['multi_layer_perceptron.md']},
|
||||
'gan': {'_i_files': ['gan.md']},
|
||||
'learn': { '_i_files': [ 'active_learning.md',
|
||||
'backpropagation_learn.md',
|
||||
'fitting.md',
|
||||
'learning_kinds.md',
|
||||
'semi-supervised_text_classification.md']},
|
||||
'llm': { '_i_files': [ 'autoregressive.md',
|
||||
'language_models.md',
|
||||
'llm.md'],
|
||||
'google': {'_i_files': ['bard.md']},
|
||||
'langchain': {'_i_files': ['langchain.md']},
|
||||
'openai': {'_i_files': ['openai.md', 'gpt4.md']},
|
||||
'oss': {'_i_files': ['vicuna.md']},
|
||||
'prompting': {'_i_files': ['cot.md']},
|
||||
'research': { '_i_files': [ 'using-llms-for-reading-thoughts.md']}},
|
||||
'multi-receptors': {'_i_files': ['nn-neurotransmitters.md']},
|
||||
'news': {'_i_files': ['google-codered.md', 'we-have-no-moat.md']},
|
||||
'rnn': { '_i_files': [ 'bci-rnn.md',
|
||||
'bionets.md',
|
||||
'echo-state-networks.md',
|
||||
'rnn.md',
|
||||
'rnn_overlearning.md',
|
||||
'rnn_timer_example.md',
|
||||
'rnn_usefulness.md',
|
||||
'rockets_rnn.md']},
|
||||
'tools': { '_i_files': [ 'chatgpt.md',
|
||||
'github-copilot.md',
|
||||
'vector-database.md',
|
||||
'bing-chat.md']}},
|
||||
'assets': {'_i_files': [], 'css': {'_i_files': ['style.scss']}},
|
||||
'attachments': { '_i_files': [ '2022-04-26-19-39-39.png',
|
||||
'Josefin_Sans.zip',
|
||||
'Montserrat.zip',
|
||||
'abawo-color-palette-v1.pdf',
|
||||
'abawo-color-palette-v1.png',
|
||||
'abawo-logo-square.png',
|
||||
'abawo-logo-square.svg',
|
||||
'abawo-logo.png',
|
||||
'abawo-logo.svg',
|
||||
'bid_ask_spread.png',
|
||||
'clustering_ni-seminar_fabian-karl_2022-02-03-12-37-28.png',
|
||||
'foam-icon.png',
|
||||
'gamedev_tutorial_properties.png',
|
||||
'gen-discr-training.svg',
|
||||
'jaeger_feedforward.png',
|
||||
'rnn-esn-talk_daniel-ruepp_ni-seminar_3.2.2022.png',
|
||||
'rnn_feedforward_net.png',
|
||||
'rnn_structure_jaeger_tutorial_timer.png',
|
||||
'viofs-folder-on-iso.png',
|
||||
'virt-manager-folder-share.png',
|
||||
'2023-05-12-12-09-30-langchain-memory.png',
|
||||
'2023-05-19-12-22-28.png',
|
||||
'cot-prompting-yaoTreeThoughtsDeliberate2023.png',
|
||||
'evalResults-yaoTreeThoughtsDeliberate2023.png',
|
||||
'evalTable-yaoTreeThoughtsDeliberate2023.png',
|
||||
'titleImg-panDragYourGAN.png',
|
||||
'tot-prompting-yaoTreeThoughtsDeliberate2023.png']},
|
||||
'cad': {'_i_files': ['CalculiX.md']},
|
||||
'cs': {'_i_files': ['reservoir-computing.md']},
|
||||
'furntiture': {'_i_files': ['element-shelving-system.md']},
|
||||
'gamedev': { '_i_files': ['prototyping.md'],
|
||||
'mechanics': {'_i_files': ['difficulty.md', 'tutorial.md']},
|
||||
'target_audience': {'_i_files': ['game_literacy.md']}},
|
||||
'helium': { '_i_files': ['helium.md'],
|
||||
'manufacturing': { '_i_files': [],
|
||||
'hs1': { '_i_files': [ 'hs1-ethernet-driver.md',
|
||||
'hs1-processor.md',
|
||||
'hs1.md']}}},
|
||||
'investing': { '_i_files': [ 'diversification.md',
|
||||
'international_investing.md',
|
||||
'value_vs_growth_stocks.md'],
|
||||
'brokers': { '_i_files': [ 'broker.md',
|
||||
'zero_commission_brokers.md']},
|
||||
'value': { '_i_files': [ 'bottom_up_market_research.md',
|
||||
'in-depth_company_research.md',
|
||||
'value_investing.md']}},
|
||||
'job_search': {'_i_files': ['ats.md', 'resume_writing.md']},
|
||||
'linux': { '_i_files': [],
|
||||
'kvm': { '_i_files': [ 'kvm-virtualization.md',
|
||||
'libvirt.md',
|
||||
'libvirtd-folder-share-windows-guest.md',
|
||||
'qemu.md',
|
||||
'virt-manager.md']}},
|
||||
'manufacturing': { '_i_files': [],
|
||||
'oem': { '_i_files': [ 'oem-in-china.md',
|
||||
'oem.md',
|
||||
'value-added-reseller.md']}},
|
||||
'maths': { '_i_files': [],
|
||||
'game_theory': { '_i_files': [ 'positive_sum_game.md',
|
||||
'zero_sum_game.md']}},
|
||||
'meditation': {'_i_files': ['shunya.md']},
|
||||
'people': {'_i_files': [], 'developers': {'_i_files': ['hwchase17.md']}},
|
||||
'psychology': {'_i_files': ['the-faschist-mindset.md']},
|
||||
'robotics': {'_i_files': ['arco.md']},
|
||||
'skills': { '_i_files': [ 'jack_of_all_trades_master_of_one.md',
|
||||
'skill_stacking.md']},
|
||||
'uni': { '_i_files': [],
|
||||
's5': { '_i_files': ['thomas-thuem-cb-pruefung-stuff.md'],
|
||||
'compilers': {'_i_files': ['lexing_anki.md']},
|
||||
'grn': {'_i_files': ['questions.md']}}}}
|
||||
46
utils.py
Normal file
46
utils.py
Normal file
@ -0,0 +1,46 @@
|
||||
from collections import defaultdict
|
||||
import os
|
||||
|
||||
from typing import DefaultDict, Dict, List
|
||||
|
||||
|
||||
def tree() -> DefaultDict:
|
||||
return defaultdict(tree)
|
||||
|
||||
|
||||
def get_last_segment(path: str) -> str:
|
||||
if path[-1] == '/':
|
||||
path = path[:-1]
|
||||
return path.split(os.sep)[-1]
|
||||
|
||||
|
||||
FILES_KEY = '_i_files'
|
||||
|
||||
|
||||
def build_file_tree(root_dir: str) -> DefaultDict:
|
||||
file_tree = tree()
|
||||
root_dir = os.path.normpath(root_dir) # Normalize the path
|
||||
|
||||
for dirpath, dirnames, files in os.walk(root_dir):
|
||||
# Get the subdirectory path relative to the root directory
|
||||
subdir = os.path.relpath(dirpath, root_dir)
|
||||
|
||||
# Split the path into components to navigate the nested dictionary
|
||||
path_components = subdir.split(os.sep)
|
||||
|
||||
# Navigate to the current subdirectory in the file tree
|
||||
current_subdir = file_tree
|
||||
for component in path_components:
|
||||
current_subdir = current_subdir[component]
|
||||
|
||||
# Add files to the current subdirectory in the file tree
|
||||
current_subdir[FILES_KEY] = files
|
||||
|
||||
return file_tree
|
||||
|
||||
|
||||
# Function to convert defaultdict to dict (for readability)
|
||||
def defaultdict_to_dict(d: defaultdict) -> dict:
|
||||
if isinstance(d, defaultdict):
|
||||
d = {k: defaultdict_to_dict(v) for k, v in d.items()}
|
||||
return d
|
||||
Loading…
Reference in New Issue
Block a user