diff options
| author | vin <git@vineetk.net> | 2025-07-19 12:36:46 -0400 |
|---|---|---|
| committer | vin <git@vineetk.net> | 2025-07-19 12:36:46 -0400 |
| commit | 950c077309aff3036bdd2ab618983089faef41de (patch) | |
| tree | 29d2a464557cce39dc05574b70f2bb0e69017ecd | |
| parent | 2db3b781c15b45bc3f0771ad2ef30d6d88352adb (diff) | |
save output in data/, not where podcasts are
| -rw-r--r-- | main.py | 18 |
1 files changed, 14 insertions, 4 deletions
| @@ -14,6 +14,8 @@ client = openai.OpenAI( | |||
| 14 | ) | 14 | ) |
| 15 | 15 | ||
| 16 | MODEL = "deepseek/deepseek-chat-v3-0324" | 16 | MODEL = "deepseek/deepseek-chat-v3-0324" |
| 17 | #MODEL = "deepseek/deepseek-chat-v3-0324:free" | ||
| 18 | |||
| 17 | SYSTEM_PROMPT_TEMPLATE = """ | 19 | SYSTEM_PROMPT_TEMPLATE = """ |
| 18 | You are an expert podcast content analyzer. Your task is to identify and extract pre-recorded dynamic advertising segments from podcast transcripts. These ads typically have a distinct tone shift, often using more direct, persuasive language, and frequently include calls to action, product mentions, or specific brand names. | 20 | You are an expert podcast content analyzer. Your task is to identify and extract pre-recorded dynamic advertising segments from podcast transcripts. These ads typically have a distinct tone shift, often using more direct, persuasive language, and frequently include calls to action, product mentions, or specific brand names. |
| 19 | 21 | ||
| @@ -258,25 +260,33 @@ def process_file(input_file: str, output_file: str): | |||
| 258 | if ad_timestamps_str.startswith("[") or "no output" in ad_timestamps_str.lower() or "no ad" in ad_timestamps_str.lower(): | 260 | if ad_timestamps_str.startswith("[") or "no output" in ad_timestamps_str.lower() or "no ad" in ad_timestamps_str.lower(): |
| 259 | ad_timestamps_str = "" | 261 | ad_timestamps_str = "" |
| 260 | 262 | ||
| 263 | |||
| 264 | promptid = os.path.dirname(input_file).split('/')[-1] + "_" | ||
| 265 | promptid += os.path.splitext(os.path.basename(output_file))[0].replace(" ", "") + f"_chunk{i}" | ||
| 266 | |||
| 261 | # Split by newlines in case the model returns multiple segments | 267 | # Split by newlines in case the model returns multiple segments |
| 262 | found_timestamps = [ts.strip() for ts in ad_timestamps_str.split('\n') if ts.strip()] | 268 | found_timestamps = [ts.strip() for ts in ad_timestamps_str.split('\n') if ts.strip()] |
| 263 | all_ad_timestamps.append({"id": f"{"".join(input_file.split(".")[:-1])}_chunk{i}", f"text": "Instruction: Identify the timestamp range of the pre-recorded ad in the following transcript. Output only the HH:MM:SS.mmm-HH:MM:SS.mmm range.\n\nTranscript:\n" + user_prompt, "target": "\n".join(found_timestamps)}) | 269 | all_ad_timestamps.append({"id": promptid, f"text": "Instruction: Identify the timestamp range of the pre-recorded ad in the following transcript. Output only the HH:MM:SS.mmm-HH:MM:SS.mmm range.\n\nTranscript:\n" + user_prompt, "target": "\n".join(found_timestamps)}) |
| 264 | 270 | ||
| 265 | with open(output_file, "w") as f: | 271 | with open(output_file, "w") as f: |
| 266 | for s in all_ad_timestamps: | 272 | for s in all_ad_timestamps: |
| 267 | f.write(json.dumps(s) + "\n") | 273 | f.write(json.dumps(s) + "\n") |
| 268 | 274 | ||
| 269 | def main(): | 275 | def main(): |
| 270 | os.makedirs("data/") | 276 | os.makedirs("data/", exist_ok=True) |
| 271 | paths = sys.argv[1:] | 277 | paths = sys.argv[1:] |
| 272 | for p in paths: | 278 | for p in paths: |
| 273 | out = "data/" + os.path.splitext(os.path.basename(p))[0] + ".jsonl" | 279 | out = "data/" |
| 280 | out += os.path.dirname(p).split('/')[-1] + "_" | ||
| 281 | out += os.path.splitext(os.path.basename(p))[0] + ".jsonl" | ||
| 274 | if os.path.isfile(out): | 282 | if os.path.isfile(out): |
| 275 | paths.remove(p) | 283 | paths.remove(p) |
| 276 | print(paths) | 284 | print(paths) |
| 277 | 285 | ||
| 278 | for p in tqdm(paths): | 286 | for p in tqdm(paths): |
| 279 | out = "data/" + os.path.splitext(os.path.basename(p))[0] + ".jsonl" | 287 | out = "data/" |
| 288 | out += os.path.dirname(p).split('/')[-1] + "_" | ||
| 289 | out += os.path.splitext(os.path.basename(p))[0] + ".jsonl" | ||
| 280 | process_file(p, out) | 290 | process_file(p, out) |
| 281 | 291 | ||
| 282 | if __name__ == "__main__": | 292 | if __name__ == "__main__": |
