podcast-sponsor-remove

Attempt at identify sponsored segments in audio transcripts and removing them.
Log | Files | Refs

length.py (770B)

      1 import json
      2 from transformers import AutoTokenizer
      3 
      4 MODEL_ID = "google/gemma-3-270m-it"
      5 DATASET_PATH = "train.jsonl"
      6 
      7 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
      8 token_lengths = []
      9 
     10 print(f"Loading dataset from {DATASET_PATH}...")
     11 with open(DATASET_PATH, 'r', encoding='utf-8') as f:
     12     for line in f:
     13         entry = json.loads(line)
     14         text = entry.get("text", "")
     15         # Tokenize the text and get the number of tokens
     16         length = len(tokenizer(text).input_ids)
     17         token_lengths.append(length)
     18 
     19 print(f"Number of examples: {len(token_lengths)}")
     20 if token_lengths:
     21     print(f"Min length:    {min(token_lengths)}")
     22     print(f"Max length:    {max(token_lengths)}")
     23     print(f"Average length:  {sum(token_lengths) / len(token_lengths):.2f}")