portal64-still-alive/tools/level_scripts/subtitle_generate.py

#!/usr/bin/env python3
import os
import re
import sys
import json
from os.path import exists

hl_gameui_whitelist = {
    "GAMEUI_GAMEMENU_RESUMEGAME",
    "GAMEUI_SAVEGAME",
    "GAMEUI_LOADGAME",
    "GAMEUI_NEWGAME",
    "GAMEUI_OPTIONS",
    "GAMEUI_GAMEMENU_QUIT",

    "GAMEUI_SOUNDEFFECTVOLUME",
    "GAMEUI_MUSICVOLUME",
    "GAMEUI_SUBTITLESANDSOUNDEFFECTS",
}

language_translations = {
	'brazilian': 'Brasileiro',
    'bulgarian': 'Български език',
    'czech': 'Čeština',
    'danish': 'Dansk',
    'german': 'Deutsch',
    'english': 'English',
    'spanish': 'Español',
	'greek': 'Ελληνικά',
    'french': 'Français',
    'italian': 'Italiano',
    'polish': 'Język polski',
    'latam': 'Latam',
    'hungarian': 'Magyar nyelv',
    'dutch': 'Nederlands',
    'norwegian': 'Norsk',
    'portuguese': 'Português',
    'russian': 'Русский язык',
    'romanian': 'Românește',
    'finnish': 'Suomi',
    'swedish': 'Svenska',
    'turkish': 'Türkçe',
	'ukrainian': 'Українська мова',
}

def get_supported_characters():
    with open('assets/fonts/dejavu_sans_book_8.json', 'r') as f:
        content = json.loads('\n'.join(f.readlines()))

    result = {' ', '\t', '\n', '\r'}
    
    for symbol in content['symbols']:
        result.update(chr(symbol['id']))

    return result


def dump_lines(sourcefile_path, lines):
    if not os.path.exists(os.path.dirname(os.path.abspath(sourcefile_path))):
        os.makedirs(os.path.dirname(os.path.abspath(sourcefile_path)))

    with open(sourcefile_path, "w") as f:
        f.writelines(lines)

def get_caption_keys_values_language(lines):
    language = "English"
    keys = []
    values = []

    found_language = False
    found_tokens = False
    for line in lines:
        if not found_language:
            if not '"Language"' in line:
                continue
            found_language = True
            line = line.replace('"', "")
            key, val = line.split()
            language = val
            continue
        if not found_tokens:
            if not '"Tokens"' in line:
                continue
            found_tokens = True
            continue
        if ("{" in line) or ("}" in line) :
            continue
        keyval= re.split('"\t+"', line)
        if len(keyval) != 2:
            keyval= line.split('"   "')
            if len(keyval) != 2:
                continue
        if "[english]" in keyval[0] or 'commentary' in keyval[0]:
            continue
        key = keyval[0].replace('"', "").replace(".", "_").replace("-", "_").replace('\\', "_").replace('#', "").upper()
        val = keyval[1].replace('"', "").replace("\n", "").replace("\\", "")
        val = re.sub(r'\<clr.+\>','',val)
        val = re.sub(r'\<norepeat.+\>','',val)
        val = val.replace("<sfx>", "")
        val = re.sub(r'\<len:.+\>','',val)
        val = val.replace("<len>", "")
        keys.append(key)
        values.append(val)
    
    return keys, values, language

def make_overall_subtitles_header(all_header_lines, languages_list, message_count, max_message_length):
    header_lines = []
    header_lines.append("#ifndef __SUBTITLES_H__\n")
    header_lines.append("#define __SUBTITLES_H__\n")
    header_lines.append("\n")
    header_lines.append(f"#define NUM_SUBTITLE_LANGUAGES {len(languages_list)}\n")
    header_lines.append(f"#define NUM_SUBTITLE_MESSAGES {message_count + 1}\n")
    header_lines.append(f"#define MAX_SUBTITLE_LENGTH  {max_message_length}\n")
    header_lines.append("\n")
    header_lines.append("struct SubtitleBlock {\n")
    header_lines.append("    char* romStart;\n")
    header_lines.append("    char* romEnd;\n")
    header_lines.append("    char** values;\n")
    header_lines.append("};\n")
    header_lines.append("\n")
    header_lines.append("extern char* SubtitleLanguages[];\n")
    header_lines.append("extern struct SubtitleBlock SubtitleLanguageBlocks[];\n")
    header_lines.append("\n")

    for idx, language in enumerate(languages_list):
        header_lines.append(f"#define LANGUAGE_{language.upper()} {idx}\n")

    header_lines.append("\n")


    if len(languages_list) > 0:
        header_lines.extend(all_header_lines)
    else:
        header_lines.append(f"enum SubtitleKey\n")
        header_lines.append("{\n")
        header_lines.append('    SubtitleKeyNone,\n')
        header_lines.append("};\n")
        header_lines.append("\n")

    header_lines.append("#endif")

    dump_lines("build/src/audio/subtitles.h", header_lines)

def make_SubtitleKey_headerlines(keys):
    header_lines = []
    header_lines.append("\n")
    header_lines.append(f"enum SubtitleKey\n")
    header_lines.append("{\n")
    header_lines.append('    SubtitleKeyNone,\n')
    for key in keys:
        header_lines.append(f'    {key},\n')
    header_lines.append("};\n")
    header_lines.append("\n")
    return header_lines

def make_subtitle_for_language(lang_lines, lang_name, keys):
    lines = []

    lines.append('#include "subtitles.h"')
    lines.append("\n")
    lines.append("\n")

    for idx, value in enumerate(lang_lines):
        lines.append(f'char __translation_{lang_name}_{keys[idx]}[] = "{value}";\n')

    lines.append("\n")
    lines.append(f"char* gSubtitle{lang_name}[NUM_SUBTITLE_MESSAGES] = {'{'}\n")

    # SubtitleKeyNone
    lines.append('    "",\n')

    for idx, value in enumerate(lang_lines):
        lines.append(f'    __translation_{lang_name}_{keys[idx]},\n')

    lines.append("};\n")

    dump_lines(f"build/src/audio/subtitles_{lang_name}.c", lines)

def determine_invalid_characters(lang_name, lang_lines, good_characters):
    used_characters = set()

    for value in lang_lines:
        used_characters = used_characters | set(value)

    invalid = used_characters - good_characters

    if len(invalid) == 0:
        return used_characters
    
    print(f"{lang_name} has {len(invalid)} invalid charcters\n{''.join(sorted(list(invalid)))}")

    return used_characters


def make_subtitle_ld(languages):
    lines = []

    for language in languages:
        language_name = language['name']

        lines.append(f"    __romPos = (__romPos + 15) & ~0xF;\n")
        lines.append(f"    BEGIN_SEG(subtitles_{language_name}, 0x04000000)\n")
        lines.append("    {\n")
        lines.append(f"       build/src/audio/subtitles_{language_name}.o(.data);\n")
        lines.append(f"       build/src/audio/subtitles_{language_name}.o(.bss);\n")
        lines.append("    }\n")
        lines.append(f"    END_SEG(subtitles_{language_name})\n")
        lines.append("\n")

    dump_lines('build/subtitles.ld', lines)

def make_overall_subtitles_sourcefile(language_list):
    sourcefile_lines = []
    sourcefile_lines.append('#include "subtitles.h"\n')

    sourcefile_lines.append("\n")
    sourcefile_lines.append("char* SubtitleLanguages[] =\n")
    sourcefile_lines.append("{\n")
    if len(language_list) <= 0:
        sourcefile_lines.append(f'    "",\n')
    else:
        for language in language_list:
            sourcefile_lines.append(f'    "{language_translations[language]}",\n')
    sourcefile_lines.append("};\n")
    sourcefile_lines.append("\n")

    for language in language_list:
        sourcefile_lines.append(f"extern char _subtitles_{language}SegmentRomStart[];\n")
        sourcefile_lines.append(f"extern char _subtitles_{language}SegmentRomEnd[];\n")
        sourcefile_lines.append(f"extern char* gSubtitle{language}[NUM_SUBTITLE_MESSAGES];\n")
        sourcefile_lines.append("\n")

    sourcefile_lines.append("struct SubtitleBlock SubtitleLanguageBlocks[] = {\n")

    for language in language_list:
        sourcefile_lines.append("    {\n")
        sourcefile_lines.append(f"        _subtitles_{language}SegmentRomStart,\n")
        sourcefile_lines.append(f"        _subtitles_{language}SegmentRomEnd,\n")
        sourcefile_lines.append(f"        gSubtitle{language},\n")
        sourcefile_lines.append("    },\n")

    sourcefile_lines.append("};\n")
    sourcefile_lines.append("\n")

    dump_lines("build/src/audio/subtitles.c", sourcefile_lines)

def read_translation_file(filepath):
    if not exists(filepath):
        return [], [], ''

    lines = []
    
    with open(filepath, "r", encoding='utf-16-le') as f:
        lines = f.readlines()
    
    new_lines = []
    for line in lines:
        line = line.replace("\x00", "")
        if "\n" != line:
            new_lines.append(line)

    return get_caption_keys_values_language(new_lines)

def filter_whitelist(keys, values, whitelist):
    result_keys = []
    result = []

    for index in range(len(keys)):
        if keys[index] in whitelist:
            result_keys.append(keys[index])
            result.append(values[index])

    return result_keys, result

def process_all_closecaption_files(dir, language_names):
    values_list = []
    header_lines = []
    language_list = []
    language_with_values_list = []
    key_order = None
    default_values = None

    for langauge_name in language_names:
        filename = f"closecaption_{langauge_name}.txt"

        try:  
            filepath = os.path.join(dir, filename)

            k,v,l = read_translation_file(filepath)

            gamepad_k, gamepad_v, _ = read_translation_file(f"vpk/Portal/hl2/resource/gameui_{langauge_name}.txt")

            gamepad_k, gamepad_v = filter_whitelist(gamepad_k, gamepad_v, hl_gameui_whitelist)

            extra_k, extra_v, _ = read_translation_file(f"assets/translations/extra_{langauge_name}.txt")

            k = k + gamepad_k + extra_k
            v = v + gamepad_v + extra_v

            if not key_order:
                header_lines = make_SubtitleKey_headerlines(k)
                key_order = k
                default_values = v
            else:
                index_mapping = {}
                for idx, x in enumerate(k):
                    index_mapping[x] = idx

                new_values = []

                for idx, key in enumerate(key_order):
                    if key in index_mapping:
                        new_values.append(v[index_mapping[key]])
                    else:
                        new_values.append(default_values[idx])

                v = new_values

            values_list.append(v)
            language_list.append(l)
            
            language_with_values_list.append({
                'value': v,
                'name': l,
            })
            print(filename, " - PASSED")
        except Exception as e:
            print(e)
            print(filename, " - FAILED")
            raise
            continue

    good_characters = get_supported_characters()
    used_characters = set()

    max_message_length = 0

    for language in language_with_values_list:
        make_subtitle_for_language(language['value'], language['name'], key_order)
        used_characters = used_characters | determine_invalid_characters(language['name'], language['value'], good_characters)

        for value in language['value']:
            max_message_length = max(max_message_length, len(value))

    print(f"needed characters\n{''.join(sorted(list(good_characters & used_characters)))}")
    print(f"unused characters\n{''.join(sorted(list(good_characters - used_characters)))}")
    print(f"invalid characters\n{''.join(sorted(list(used_characters - good_characters)))}")
    print(f"max message length\n{max_message_length}")

    make_subtitle_ld(language_with_values_list)

    make_overall_subtitles_header(header_lines, language_list, len(language_with_values_list[0]['value']), max_message_length)
    make_overall_subtitles_sourcefile(language_list)



process_all_closecaption_files("vpk/Portal/portal/resource", sys.argv[1:])