Feature request: added json config

This commit is contained in:
2023-10-10 14:17:38 +02:00
parent 101f2db788
commit e2b5ad15b4
4 changed files with 41 additions and 14 deletions
+6 -2
View File
@@ -52,14 +52,18 @@ Jobs specify tasks for transcript creation. A job contains the following informa
- `name`: string: The name of the job - `name`: string: The name of the job
- `source`: List[string]: The sources of the videos (paths, YouTube links, TibAV links) - `source`: List[string]: The sources of the videos (paths, YouTube links, TibAV links)
- `modelpath`: Optional[string]: The path to the model if a specific model should be used.
- `language`: Optional[string]: The language of the video (If not specified employs auto-detection) - `language`: Optional[string]: The language of the video (If not specified employs auto-detection)
## to-do ## to-do
### Feature: specific model
add an opitonal model path to the json config
- `model_path`:Optional[string]: The path to a specific language model to be used (mainly for testing)
### Other
- seperate data production form code - seperate data production form code
- remember to accept the conditions on models if you use huggingface - remember to accept the conditions on models if you use huggingface
- do a config json
- check warnings - check warnings
- downloader for tib-av - downloader for tib-av
- more optional setting - more optional setting
+6
View File
@@ -0,0 +1,6 @@
{
"name": "myjob1",
"source": [
"https://www.youtube.com/watch?v=BVVo1BaL6JA"
]
}
+22 -6
View File
@@ -1,20 +1,36 @@
import os import os
import json
import argparse
from src.transcript_generator import TranscriptGenerator from src.transcript_generator import TranscriptGenerator
def main(): def main():
# input could be a YouTube link or a local file # Create a parser
input_sources = ["https://www.youtube.com/watch?v=BVVo1BaL6JA"] parser = argparse.ArgumentParser(description='Transcript Generator for Videos')
# Add the arguments
parser.add_argument('--job', type=str, help='The name of the job', required=True)
# Parse the arguments
args = parser.parse_args()
# Load the job config
with open(f'jobs/{args.job}.json', 'r') as f:
job_config = json.load(f)
# Check the hf_token
hf_token = os.environ.get("HF_TOKEN") hf_token = os.environ.get("HF_TOKEN")
if not hf_token: if not hf_token:
hf_token = input("Please enter your HF_TOKEN: ") hf_token = input("Please enter your HF_TOKEN: ")
os.environ["HF_TOKEN"] = hf_token os.environ["HF_TOKEN"] = hf_token
transcript_generator = TranscriptGenerator() transcript_generator = TranscriptGenerator(
transcript_generator.generate(input_sources) name=job_config['name'],
language_code=job_config['language_code']
)
transcript_generator.generate(job_config['source'])
if __name__ == "__main__": if __name__ == "__main__":
main() main()
+7 -6
View File
@@ -15,9 +15,10 @@ class TranscriptGenerator:
to generate transcriptions from given input sources. to generate transcriptions from given input sources.
""" """
def __init__(self): def __init__(self, job_name, language_code="en"):
self.file_manager = FileManager() self.file_manager = FileManager()
self.transcriber = Transcriber() self.transcriber = Transcriber(language_code)
self.job_name = job_name
def generate(self, input_sources: List[str]) -> None: def generate(self, input_sources: List[str]) -> None:
""" """
@@ -29,15 +30,15 @@ class TranscriptGenerator:
for i, source in enumerate(input_sources): for i, source in enumerate(input_sources):
try: try:
if source.startswith("http"): if source.startswith("http"):
mp4_filename = f"audio_{i + 1}.mp4" mp4_filename = f"audio_{self.job_name}_{i + 1}.mp4"
wav_filename = f"audio_{i + 1}.wav" wav_filename = f"audio_{self.job_name}_{i + 1}.wav"
self.file_manager.download_youtube_audio(source, mp4_filename) self.file_manager.download_youtube_audio(source, mp4_filename)
self.file_manager.convert_to_wav(mp4_filename, wav_filename) self.file_manager.convert_to_wav(mp4_filename, wav_filename)
else: else:
wav_filename = self.file_manager.load_local_file(source) wav_filename = self.file_manager.load_local_file(source)
self.transcriber.transcribe(wav_filename) self.transcriber.transcribe(wav_filename)
json_output_file = f"result_{i + 1}.json" json_output_file = f"result_{self.job_name}_{i + 1}.json"
txt_output_file = f"transcripts_{i + 1}.txt" txt_output_file = f"transcripts_{self.job_name}_{i + 1}.txt"
self.transcriber.save_transcript(json_output_file, txt_output_file) self.transcriber.save_transcript(json_output_file, txt_output_file)
except Exception as e: except Exception as e:
logger.exception( logger.exception(