Add files via upload

00_exportchapters.py

@@ -0,0 +1,83 @@
+#Confirmed working as of 1/22/23
+
+import os
+import shutil
+from pydub import AudioSegment
+
+AudioSegment.converter = "/opt/homebrew/Cellar/ffmpeg/5.1.2_3/bin/ffmpeg"
+AudioSegment.ffmpeg = "/opt/homebrew/Cellar/ffmpeg/5.1.2_3/bin/ffmpeg"
+AudioSegment.ffprobe ="/opt/homebrew/Cellar/ffmpeg/5.1.2_3/bin/ffmpeg"
+
+def export_audio_file(start, end, name):
+    start_ms = int(start)
+    end_ms = int(end)
+    export = combined[start_ms:end_ms]
+    folder_name = folder.split(" - ")[-1].strip()
+    destination_path = '/Users/jonas/Documents/SERVER/BOOKS/01_LABELED AUDIO'
+    author_name = folder.split(" - ")[0].strip()
+    export_folder = os.path.join(destination_path, author_name, folder_name)
+    author_folder = os.path.join(destination_path, author_name)
+    if not os.path.exists(author_folder):
+        os.makedirs(author_folder)
+        export_folder = os.path.join(author_folder, folder_name)
+
+    if not os.path.exists(export_folder):
+        os.makedirs(export_folder)
+    export_path = os.path.join(export_folder, f"{name}.mp3")
+    export.export(export_path, format="mp3")
+
+    metadata_file = os.path.join(folder_path, "cleaned_metadata.json")
+    chapters_file = os.path.join(folder_path, "overdrive_chapters_ms.txt")
+    album_art = os.path.join(folder_path, "folder.jpg")
+    if os.path.exists(album_art):
+        shutil.copy(album_art, export_folder)
+    shutil.copy(metadata_file, export_folder)
+
+main_directory = "/Users/jonas/Documents/SERVER/BOOKS/TEST"
+
+for folder in sorted(os.listdir(main_directory)):
+    folder_path = os.path.join(main_directory, folder)
+    if os.path.isdir(folder_path):
+        try:
+            # Import MP3s in alphabetical order
+            mp3_files = sorted(os.listdir(folder_path))
+            mp3_files = [os.path.join(folder_path, file) for file in mp3_files if file.endswith(".mp3")]
+            
+            combined = AudioSegment.empty()
+            for mp3_file in mp3_files:
+                print(mp3_file)
+                combined += AudioSegment.from_file(mp3_file)
+
+            # Align end to end
+            combined = combined.set_channels(1)
+
+            # Import labels
+            labels_file = os.path.join(folder_path, "overdrive_chapters_ms_spans.txt")
+            labels = []
+            with open(labels_file, "r") as f:
+                for line in f:
+                    start, end, name = line.strip().split("\t")
+                    labels.append((start, end, name))
+
+            # Initialize counter for duplicate label names
+            counter = {}
+
+            # Initialize counter for file export
+            file_count = 0
+
+            for i, label in enumerate(labels):
+                start, end, name = label
+                if end == "0" and i == len(labels) - 1:
+                    end = combined.duration_seconds * 1000
+
+                file_count += 1
+                name = f"{file_count:03}_{name.replace('/', '_')}" 
+                # format file_count as 3 digit number
+                export_audio_file(start, end, name)
+
+        except Exception as e:
+            print(f"Error processing {folder}: {e}")
+    
+    #Archive everything except chapters_list.py
+    if folder_path != "chapters_list.py":
+        shutil.move(folder_path, '/Users/jonas/Documents/SERVER/BOOKS/00_ARCHIVE/' + folder)

Order of operations.txt

@@ -0,0 +1,12 @@
+Order of operations:
+
+01: Download .odm files
+02: Use sh overdrivedelete.sh to unpack
+03: Use chapters_list.py to get the chapters times and titles
+04: Use metadatatoxml.py to get xml
+05: Use xmlparse.py to clean metadata
+06: Use chapter_ms.py to put the chapter times in ms
+07: Use durations.py to create the duration spans in the chapter file
+08: Use 00_exportchapters.py to get the labeled audio, pull the metadata txt & folder.jpg into the new folder, and move the old folder to archive
+09: Use final_metadata_add.py to add metadata to the tracks and move the author folders to 02_READY FOR PLEX
+10: Drag to Plex -- the odm download and this part are the two manuals components and I consider those safeguards. Worst case scenario if something went wrong you'd probably see it during this step and you could revert to the archives.

README.md

@@ -0,0 +1,2 @@
+# odm-plex
+Automates the process of opening ODM's and formatting them for use in Plex including editing down to chapters and adding metadata.

chapter_ms.py

@@ -0,0 +1,33 @@
+#Confirmed working as of 1/22/23
+
+import os
+import re
+from datetime import datetime, timedelta
+
+def duration_to_milliseconds(duration):
+    hours, minutes, seconds = duration.split(':')
+    seconds = int(hours)*3600 + int(minutes)*60 + float(seconds)
+    return int(seconds * 1000)
+
+# path to directory containing the files
+directory = '/Users/jonas/Documents/SERVER/BOOKS/TEST'
+
+# loop through all files in directory tree
+for dirpath, dirnames, filenames in os.walk(directory):
+    for filename in filenames:
+        if filename.startswith("overdrive_chapters") and filename.endswith(".txt"):
+            # open the file and read the contents
+            with open(os.path.join(dirpath, filename), 'r') as f:
+                lines = f.readlines()
+
+            # create a new file with the same name but with "milliseconds" added to the end
+            new_filename = filename[:-4] + "_ms.txt"
+            with open(os.path.join(dirpath, new_filename), 'w') as new_file:
+                # loop through each line in the file
+                for line in lines:
+                    # split the line into duration and label using regular expression
+                    duration, label = re.split(r"\s+", line.strip(), 1)
+                    # convert the duration to milliseconds
+                    milliseconds = duration_to_milliseconds(duration)
+                    # write the converted duration and label to the new file
+                    new_file.write(f'{milliseconds} {label}\n')

durations.py

@@ -0,0 +1,36 @@
+#Confirmed working as of 1/22/23
+
+import os
+
+root_dir = '/Users/jonas/Documents/SERVER/BOOKS/TEST'
+for subdir, dirs, files in os.walk(root_dir):
+    for file in files:
+        file_path = os.path.join(subdir, file)
+        if file == 'overdrive_chapters_ms.txt':
+            with open(file_path, "r") as file:
+                lines = file.read().split('\n')
+            new_file_path = file_path.replace('.txt', '_spans.txt')
+            with open(new_file_path, "w") as file:
+                for i in range(len(lines)):
+                    parts = lines[i].split(" ")
+                    if i == len(lines) - 2:
+                        # If it's the second-to-last line, write the last line's number and 0
+                        file.write(parts[0])
+                        file.write("\t" + lines[i + 1].split(" ")[0])
+                        file.write("0")
+                    elif i != len(lines) - 1:
+                        # Otherwise, write the number from the next line to the file
+                        file.write(parts[0])
+                        file.write("\t" + lines[i + 1].split(" ")[0])
+                    else:
+                        # If it's the last line, just write the number and 0
+                        if parts[0] != "":
+                            file.write(parts[0])
+                            file.write("0")
+                    # Write the rest of the line to the file
+                    file.write("\t" + " ".join(parts[1:]))
+                    if i != len(lines) - 1 or (i == len(lines) - 1 and lines[i] != ""):
+                        file.write("\n")
+                if lines[-1] == "":
+                    file.seek(file.tell()-1, os.SEEK_SET)
+                    file.truncate()

extract_overdrive_chapters.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# Credit to ex-nerd
+#
+# Recursively scans current or specified directory for all subdirectories
+# containing mp3 files. If these mp3 files contain overdrive chapter markers
+# (id3 tag), writes overdrive_chapters.txt to the same directory.
+#
+# Usage:
+#
+# extract_overdrive_chapters.py [optional directory path]
+#
+# Use with build_m4b from https://github.com/ex-nerd/audiotools
+#
+# Note: Due to overdrive low quality, there is no point in encoding aac files
+# with better than: 64kbps stereo, HE, optimize for voice
+#
+
+import os, sys, re
+import mutagen.id3 as id3
+from mutagen.mp3 import MP3
+from mutagen import File
+
+from collections import OrderedDict
+
+def timestr(secs):
+    (secs, ms) = str(secs).split(".")
+    ms = float(ms[0:3] + "." + ms[3:])
+    secs = int(secs)
+    hours = int(secs // 3600)
+    secs = secs % 3600
+    mins = int(secs // 60)
+    secs = secs % 60
+    return f"{hours:02}:{mins:02}:{secs:02}.{ms:03.0f}"
+
+
+def load_mp3(total, dir, file):
+    path = os.path.join(dir, file)
+    audio = MP3(path)
+    # print(audio.info.length)  # , audio.info.bitrate
+    m = id3.ID3(path)
+
+    data = m.get("TXXX:OverDrive MediaMarkers")
+    if not data:
+        print("Can't find TXXX data point for {0}".format(file))
+        print(m.keys())
+        return
+    info = data.text[0]
+    file_chapters = re.findall(
+        r"<Name>\s*([^>]+?)\s*</Name><Time>\s*([\d:.]+)\s*</Time>", info, re.MULTILINE
+    )
+    chapters = []
+    for chapter in file_chapters:
+        (name, length) = chapter
+        name = re.sub(r'^"(.+)"$', r"\1", name)
+        name = re.sub(r"^\*(.+)\*$", r"\1", name)
+        name = re.sub(
+            r"\s*\([^)]*\)$", "", name
+        )  # ignore any sub-chapter markers from Overdrive
+        name = re.sub(
+            r"\s+\(?continued\)?$", "", name
+        )  # ignore any sub-chapter markers from Overdrive
+        name = re.sub(
+            r"\s+-\s*$", "", name
+        )  # ignore any sub-chapter markers from Overdrive
+        name = re.sub(
+            r"^Dis[kc]\s+\d+\W*$", "", name
+        )  # ignore any disk markers from Overdrive
+        name = name.strip()
+        t_parts = list(length.split(":"))
+        t_parts.reverse()
+        seconds = total + float(t_parts[0])
+        if len(t_parts) > 1:
+            seconds += int(t_parts[1]) * 60
+        if len(t_parts) > 2:
+            seconds += int(t_parts[2]) * 60 * 60
+        chapters.append([name, seconds])
+        # print(name, seconds)
+    return (total + audio.info.length, chapters)
+
+
+def visit(dirname, filenames):
+    print(dirname)
+    os.chdir(dirname)
+    # Parse the files
+    total = 0
+    all_chapters = OrderedDict()
+    for file in sorted(filenames):
+        if file.endswith(".mp3"):
+            (total, chapters) = load_mp3(total, dirname, file)
+            # print(repr(chapters))
+            for chapter in chapters:
+                if chapter[0] in all_chapters.keys():
+                    continue
+                all_chapters[chapter[0]] = chapter[1]
+    if len(all_chapters) > 0:
+        with open("overdrive_chapters.txt", "w") as file:
+            for name, length in all_chapters.items():
+                chapstr = f"{timestr(length)} {name}"
+                print(chapstr)
+                file.write(chapstr + "\n")
+    # print(repr(all_chapters))
+
+
+if __name__ == "__main__":
+
+    if len(sys.argv) > 1:
+        path = os.path.abspath(sys.argv[1])
+    else:
+        path = os.path.abspath(".")
+
+    for dirname, dirs, files in os.walk(path, topdown=True):
+        dirs[:] = [d for d in dirs if d not in {".git", ".direnv"}]
+        visit(dirname, files)

final_metadata_add.py

@@ -0,0 +1,78 @@
+#Confirmed working as of 1/23/23
+
+import os
+import shutil
+import json
+import re
+from mutagen.id3 import ID3, APIC, TIT2, TPE1, TCOM, TCON, TSOA, TRCK, TIT3, TALB, COMM
+
+def traverse_directory(directory):
+    # Iterate through all subfolders and files
+    for root, dirs, files in os.walk(directory):
+        for dir in dirs:
+            subfolder_path = os.path.join(root, dir)
+            if os.path.exists(os.path.join(subfolder_path, "cleaned_metadata.json")):
+                add_metadata(subfolder_path)
+
+def add_metadata(directory):
+    # Get the metadata from the cleaned_metadata.json file in the current subfolder
+    with open(os.path.join(directory, "cleaned_metadata.json"), "r") as f:
+        json_data = json.load(f)
+        artist = json_data.get("Author", "")
+        composer = json_data.get("Narrator", "")
+        genre = json_data.get("Subjects", "")
+        album_title = json_data.get("Title", "")
+
+    # Rest of the code
+
+    # Replace all occurrences of -s with 's in the album title
+    album_title = re.sub("-s", "'s", album_title)
+
+    # Get a list of all mp3 files in the current directory
+    mp3_files = [f for f in os.listdir(directory) if f.endswith(".mp3")]
+
+    # Sort the list of mp3 files alphabetically
+    mp3_files.sort()
+
+    # Iterate through the list of mp3 files
+    for i, mp3_file in enumerate(mp3_files, 1):
+        mp3_file_path = os.path.join(directory, mp3_file)
+
+        # Extract the title of the mp3 file
+        file_name, file_ext = os.path.splitext(mp3_file)
+        title = file_name.lstrip("0123456789_")
+
+        # Add the track number to the file using ID3
+        audio = ID3(mp3_file_path)
+        audio.add(TRCK(encoding=3, text=str(i)))
+        audio.add(TIT2(encoding=3, text=title))
+        audio.save()
+
+        # Add the metadata to the file using ID3
+        audio = ID3(mp3_file_path)
+        audio.add(TPE1(encoding=3, text=artist))
+        audio.add(TCOM(encoding=3, text=composer))
+        audio.add(TCON(encoding=3, text=genre))
+        audio.add(TALB(encoding=3, text=album_title))
+        audio.save()
+
+        # Check if folder.jpg exists in the current directory
+        if os.path.exists(os.path.join(directory, "folder.jpg")):
+            # Add the album art to the file using ID3
+            with open(os.path.join(directory, "folder.jpg"), "rb") as albumart:
+                audio = ID3(mp3_file_path)
+                audio.add(APIC(encoding=3, mime='image/jpeg', type=3, desc='Cover', data=albumart.read()))
+                audio.save()
+
+# Point the script at a directory
+directory = "/Users/jonas/Documents/SERVER/BOOKS/01_LABELED AUDIO"
+
+# Traverse the directory and apply the metadata to the mp3 files
+traverse_directory(directory)
+
+for root, dirs, files in os.walk(directory):
+    for dir in dirs:
+        subfolder_path = os.path.join(root, dir)
+        parent_folder = os.path.dirname(subfolder_path)
+        if parent_folder == directory:
+            shutil.move(subfolder_path, "/Users/jonas/Documents/SERVER/BOOKS/02_READY FOR PLEX")

metadatatoxml.py

@@ -0,0 +1,14 @@
+#Confirmed working as of 1/22/23
+
+import os
+import shutil
+
+root_dir = '/Users/jonas/Documents/SERVER/BOOKS/TEST'
+
+for subdir, dirs, files in os.walk(root_dir):
+    for file in files:
+        if file.endswith('.odm.metadata'):
+            old_file = os.path.join(subdir, file)
+            new_file = os.path.join(subdir, "metadata.xml")
+            shutil.copy2(old_file, new_file)
+            break

overdrive_full.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#download odm's
+
+# NOTE: The odm's come with metadata -- for the future system use that file because it's more reliable.
+
+# navigate to the folder where the first script is located
+cd /Users/jonas/Documents/SERVER/FRAMEWORK
+
+# unpack odm's
+bash overdrivedelete.sh
+
+# navigate to the folder where the second script is located
+cd /Users/jonas/Documents/SERVER/BOOKS/TEST
+
+# extract chapters
+python3.10 extract_overdrive_chapters.py /Users/jonas/Documents/SERVER/BOOKS/01_ANNEX
+
+# navigate back to the directory where the shell script is located
+cd /Users/jonas/Documents/SERVER/LOCKED_CODE
+
+# clean metadata
+#python3.10 00_format_metadata.py
+python3.10 metadatatoxml.py
+
+python3.10 xmlparse.py
+
+# turn chapters into ms
+python3.10 chapter_ms.py
+
+# format durations
+python3.10 durations.py
+
+# export labeled audio and archive original folder 
+# FIX: MAKE SURE IT CAN ADD TO EXISTING AUTHOR FOLDER
+python3.10 00_exportchapters.py
+
+# tag the exported audio with proper metadata
+python3.10 final_metadata_add.py
+
+#manually add to Plex

overdrivedelete.sh

@@ -0,0 +1,41 @@
+#This officially works for the full ODM extraction as of 1/21/23 at 9:54pm
+
+input_folder="/Users/jonas/Documents/SERVER/QUEUE"
+output_folder="/Users/jonas/Documents/SERVER/BOOKS/01_ANNEX"
+
+# Open ODM's in temporary directory and export contents
+for file in $input_folder/*.odm
+do
+    # Make temp dir in loop so that it clears even if there's any errors
+    temp_dir=$(mktemp -d)
+    cd "$temp_dir"
+
+    echo "folder path: $input_folder"
+    echo "temp_dir path: $temp_dir"
+
+    while true; do
+        if ~/.local/bin/overdrive download $file; then
+            break
+        fi
+        sleep 2 # wait for 2 seconds before retrying
+    done
+
+    # Get the name of the downloaded folder
+    downloaded_folder=$(find "$temp_dir" -type d -mindepth 1 -maxdepth 1 | head -n 1)
+    echo "downloaded_folder: $downloaded_folder"
+
+    # Move the downloaded folder to the destination
+    mv "$downloaded_folder" "$output_folder"
+
+    # Move the .odm.metadata to the downloaded folder
+    metadata_file=$(find "$input_folder" -name "*.odm.metadata")
+    mv "$metadata_file" "$output_folder/$(basename "$downloaded_folder")"
+
+    # Delete leftover .odm and .odm.license files
+    odm=$(find "$input_folder" -name "*.odm")
+    license=$(find "$input_folder" -name "*.odm.license")
+    rm $odm
+    rm $license
+
+    rm -rf "$temp_dir"
+done

xmlparse.py

@@ -0,0 +1,39 @@
+#Confirmed as of 1/22/23
+
+import os
+import xml.etree.ElementTree as ET
+import json
+
+def extract_metadata(xml_file):
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
+    data = {}
+    for child in root:
+        if child.tag == "Title":
+            data["Title"] = child.text
+        elif child.tag == "Creators":
+            for creator in child:
+                if creator.attrib["role"] == "Author":
+                    data["Author"] = creator.text
+                elif creator.attrib["role"] == "Narrator":
+                    data["Narrator"] = creator.text
+        elif child.tag == "Subjects":
+            subjects = []
+            for subject in child:
+                subjects.append(subject.text)
+            data["Subjects"] = ", ".join(subjects)
+    return data
+
+def process_folder(folder_path):
+    for dirpath, dirnames, filenames in os.walk(folder_path):
+        for filename in filenames:
+            if filename == "metadata.xml":
+                xml_file = os.path.join(dirpath, filename)
+                metadata = extract_metadata(xml_file)
+                cleaned_file = os.path.join(dirpath, "cleaned_metadata.json")
+                with open(cleaned_file, "w") as f:
+                    json.dump(metadata, f)
+                print(f"Processed {xml_file}, output saved to {cleaned_file}")
+
+folder_path = "/Users/jonas/Documents/SERVER/BOOKS/TEST"
+process_folder(folder_path)