Add support for fonts with more than 256 chracters

Makefile

@@ -503,6 +503,7 @@ SUBTITLE_LANGUAGES = english \
     finnish \
     french \
     german \
+	greek \
     hungarian \
     italian \
     latam \
@@ -513,8 +514,8 @@ SUBTITLE_LANGUAGES = english \
     russian \
     spanish \
     swedish \
-    thai \
+    turkish \
-    turkish
+	ukrainian
 
 SUBTITLE_SOURCES = $(SUBTITLE_LANGUAGES:%=build/src/audio/subtitles_%.c)
 SUBTITLE_OBJECTS = $(SUBTITLE_LANGUAGES:%=build/src/audio/subtitles_%.o)

assets/fonts/font_characters.txt

@@ -0,0 +1,5 @@
+ /¡¿!%'()*,-.0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]abcdefghijklmnopqrstuvwxyz–—‘’“”„…
+ÀÁÂÃÄÅÆÇÈÉÊÍÎÑÓÔÖØÚÜÝßàáâãäåæçèéêëìíîïñòóôõöøùúûüýĂ㥹ćČčďĘęĚěĞğİıŁłŃń
+ŇňŐőŘřŚśŞşŠšŢţťůűźŻżŽžȘșțΆΈΉΊΌΏΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΩάέήίΰαβγδεζηθικλ
+μνξοπρςστυφχψωϊϋόύώЄІЇАБВГДЕЖЗИЙКЛМНОПРСТУФХЦ«°·»
+ЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяёєіїѝҐґ

assets/materials/ui.skm.yaml

@@ -10,9 +10,69 @@ materials:
 
     sortOrder: 10
 
-  dejavu_sans:
+  dejavu_sans_0:
     gDPSetTile:
-      filename: "../fonts/dejavu_sans.png"
+      filename: "../fonts/dejavu_sans_0.png"
+      siz: G_IM_SIZ_4b
+      fmt: G_IM_FMT_I
+    gDPSetCombineMode: 
+      color: ["0", "0", "0", ENVIRONMENT]
+      alpha: [TEXEL0, "0", ENVIRONMENT, "0"]
+    gDPSetRenderMode: G_RM_XLU_SURF
+    gDPSetEnvColor:
+      r: 255
+      g: 255
+      b: 255
+      a: 255
+
+  dejavu_sans_1:
+    gDPSetTile:
+      filename: "../fonts/dejavu_sans_1.png"
+      siz: G_IM_SIZ_4b
+      fmt: G_IM_FMT_I
+    gDPSetCombineMode: 
+      color: ["0", "0", "0", ENVIRONMENT]
+      alpha: [TEXEL0, "0", ENVIRONMENT, "0"]
+    gDPSetRenderMode: G_RM_XLU_SURF
+    gDPSetEnvColor:
+      r: 255
+      g: 255
+      b: 255
+      a: 255
+
+  dejavu_sans_2:
+    gDPSetTile:
+      filename: "../fonts/dejavu_sans_2.png"
+      siz: G_IM_SIZ_4b
+      fmt: G_IM_FMT_I
+    gDPSetCombineMode: 
+      color: ["0", "0", "0", ENVIRONMENT]
+      alpha: [TEXEL0, "0", ENVIRONMENT, "0"]
+    gDPSetRenderMode: G_RM_XLU_SURF
+    gDPSetEnvColor:
+      r: 255
+      g: 255
+      b: 255
+      a: 255
+
+  dejavu_sans_3:
+    gDPSetTile:
+      filename: "../fonts/dejavu_sans_3.png"
+      siz: G_IM_SIZ_4b
+      fmt: G_IM_FMT_I
+    gDPSetCombineMode: 
+      color: ["0", "0", "0", ENVIRONMENT]
+      alpha: [TEXEL0, "0", ENVIRONMENT, "0"]
+    gDPSetRenderMode: G_RM_XLU_SURF
+    gDPSetEnvColor:
+      r: 255
+      g: 255
+      b: 255
+      a: 255
+
+  dejavu_sans_4:
+    gDPSetTile:
+      filename: "../fonts/dejavu_sans_4.png"
       siz: G_IM_SIZ_4b
       fmt: G_IM_FMT_I
     gDPSetCombineMode: 

src/font/font.c

@@ -1,8 +1,8 @@
 #include "font.h"
 
-int fontDetermineKerning(struct Font* font, char first, char second) {
+int fontDetermineKerning(struct Font* font, short first, short second) {
     unsigned index = ((unsigned)first * (unsigned)font->kerningMultiplier + (unsigned)second) & (unsigned)font->kerningMask;
-    int maxIterations = font->maxCollisions;
+    int maxIterations = font->kerningMaxCollisions;
 
     do {
         struct FontKerning* kerning = &font->kerning[index];
@@ -22,6 +22,28 @@ int fontDetermineKerning(struct Font* font, char first, char second) {
     return 0;
 }
 
+struct FontSymbol* fontFindSymbol(struct Font* font, short id) {
+    unsigned index = ((unsigned)id * (unsigned)font->symbolMultiplier) & (unsigned)font->kerningMask;
+    int maxIterations = font->symbolMaxCollisions;
+
+    do {
+        struct FontSymbol* symbol = &font->symbols[index];
+
+        if (symbol->id == 0) {
+            return NULL;
+        }
+
+        if (symbol->id == id) {
+            return symbol;
+        }
+        
+        ++index;
+        --maxIterations;
+    } while (maxIterations >= 0);
+
+    return NULL;
+}
+
 Gfx* fontRender(struct Font* font, char* message, int x, int y, Gfx* dl) {
     int startX = x;
     char prev = 0;
@@ -35,12 +57,13 @@ Gfx* fontRender(struct Font* font, char* message, int x, int y, Gfx* dl) {
             continue;
         }
 
-        if ((unsigned char)curr >= font->symbolCount) {
+        // TODO utf-8 decode
+        struct FontSymbol* symbol = fontFindSymbol(font, (short)curr);
+
+        if (!symbol) {
             continue;
         }
 
-        struct FontSymbol* symbol = &font->symbols[(int)curr];
-
         x += fontDetermineKerning(font, prev, curr);
 
         int finalX = x + symbol->xoffset;
@@ -72,7 +95,10 @@ int fontCountGfx(struct Font* font, char* message) {
             continue;
         }
 
-        if ((unsigned char)curr >= font->symbolCount) {
+        // TODO utf-8 decode
+        struct FontSymbol* symbol = fontFindSymbol(font, (short)curr);
+
+        if (!symbol) {
             continue;
         }
 
@@ -101,12 +127,13 @@ struct Vector2s16 fontMeasure(struct Font* font, char* message) {
             continue;
         }
 
-        if ((unsigned char)curr >= font->symbolCount) {
+        // TODO utf-8 decode
+        struct FontSymbol* symbol = fontFindSymbol(font, (short)curr);
+
+        if (!symbol) {
             continue;
         }
 
-        struct FontSymbol* symbol = &font->symbols[(int)curr];
-
         x += fontDetermineKerning(font, prev, curr);
         x += symbol->xadvance;
 

src/font/font.h

@@ -6,32 +6,51 @@
 
 struct FontKerning {
     char amount;
-    char first;
+    short first;
-    char second;
+    short second;
 };
 
 struct FontSymbol {
+    short id;
     char x, y;
     char width, height;
     char xoffset, yoffset;
     char xadvance;
+    char textureIndex;
 };
 
 struct Font {
     struct FontKerning* kerning;
     struct FontSymbol* symbols;
+    Gfx* images;
 
     char base;
     char charHeight;
-    unsigned short symbolCount;
+    unsigned short symbolMultiplier;
+    unsigned short symbolMask;
+    unsigned short symbolMaxCollisions;
+
     unsigned short kerningMultiplier;
     unsigned short kerningMask;
-    unsigned short maxCollisions;
+    unsigned short kerningMaxCollisions;
+};
+
+struct SymbolLocation {
+    short x;
+    short y;
+    short symbolIndex;
 };
 
-int fontDetermineKerning(struct Font* font, char first, char second);
 Gfx* fontRender(struct Font* font, char* message, int x, int y, Gfx* dl);
 int fontCountGfx(struct Font* font, char* message);
 struct Vector2s16 fontMeasure(struct Font* font, char* message);
 
+struct FontRenderer {
+    struct SymbolLocation symbols[128];
+    short currentSymbol;
+};
+
+void fontRendererRender(struct FontRenderer* renderer, struct Font* font, char* message, int x, int y, int maxWidth);
+Gfx* fontRendererBuildGfx(struct FontRenderer* renderer, struct Font* font, Gfx* gfx);
+
 #endif

src/menu/audio_options.c

@@ -245,7 +245,7 @@ void audioOptionsRender(struct AudioOptions* audioOptions, struct RenderState* r
 
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_ENV_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
 
     gDPPipeSync(renderState->dl++);
     menuSetRenderColor(renderState, audioOptions->selectedItem == AudioOptionGameVolume, &gSelectionGray, &gColorWhite);
@@ -279,5 +279,5 @@ void audioOptionsRender(struct AudioOptions* audioOptions, struct RenderState* r
     menuSetRenderColor(renderState, audioOptions->selectedItem == AudioOptionAudioLanguage, &gSelectionGray, &gColorWhite);
     gSPDisplayList(renderState->dl++, audioOptions->audioLanguageDynamicText);
 
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 }

src/menu/controls.c

@@ -407,7 +407,7 @@ void controlsMenuRender(struct ControlsMenu* controlsMenu, struct RenderState* r
     gDPSetScissor(renderState->dl++, G_SC_NON_INTERLACE, CONTROLS_X, CONTROLS_Y, CONTROLS_X + CONTROLS_WIDTH, CONTROLS_Y + CONTROLS_HEIGHT);
 
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
 
     gDPPipeSync(renderState->dl++);
     gDPSetScissor(renderState->dl++, G_SC_NON_INTERLACE, 0, 0, SCREEN_WD, SCREEN_HT);
@@ -437,7 +437,7 @@ void controlsMenuRender(struct ControlsMenu* controlsMenu, struct RenderState* r
         renderStateInlineBranch(renderState, controlsMenu->headers[i].headerText);
     }
 
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 
     gSPDisplayList(renderState->dl++, ui_material_list[BUTTON_ICONS_INDEX]);
     for (int i = 0; i < ControllerActionCount; ++i) {
@@ -495,7 +495,7 @@ void controlsRenderPrompt(enum ControllerAction action, char* message, float opa
     );
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_TRANSPARENT_OVERLAY_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
     gDPSetEnvColor(renderState->dl++, 232, 206, 80, opacityAsInt);
     renderState->dl = fontRender(
         &gDejaVuSansFont, 
@@ -504,7 +504,7 @@ void controlsRenderPrompt(enum ControllerAction action, char* message, float opa
         textPositionY, 
         renderState->dl
     );
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 
     gSPDisplayList(renderState->dl++, ui_material_list[BUTTON_ICONS_INDEX]);
     gDPSetEnvColor(renderState->dl++, 232, 206, 80, opacityAsInt);
@@ -545,7 +545,7 @@ void controlsRenderSubtitle(char* message, float textOpacity, float backgroundOp
     );
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_TRANSPARENT_OVERLAY_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
     if (subtitleType == SubtitleTypeCloseCaption){
         gDPSetEnvColor(renderState->dl++, 255, 140, 155, textOpacityAsInt);
     } else if (subtitleType == SubtitleTypeCaption){
@@ -559,5 +559,5 @@ void controlsRenderSubtitle(char* message, float textOpacity, float backgroundOp
         textPositionY, 
         renderState->dl
     );
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 }

src/menu/gameplay_options.c

@@ -201,7 +201,7 @@ void gameplayOptionsRender(struct GameplayOptions* gameplayOptions, struct Rende
 
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_ENV_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
 
     gDPPipeSync(renderState->dl++);
     menuSetRenderColor(renderState, gameplayOptions->selectedItem == GameplayOptionMovingPortals, &gSelectionGray, &gColorWhite);
@@ -220,5 +220,5 @@ void gameplayOptionsRender(struct GameplayOptions* gameplayOptions, struct Rende
     menuSetRenderColor(renderState, gameplayOptions->selectedItem == GameplayOptionPortalRenderDepth, &gSelectionGray, &gColorWhite);
     gSPDisplayList(renderState->dl++, gameplayOptions->portalRenderDepthText);
 
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 }

src/menu/joystick_options.c

@@ -199,7 +199,7 @@ void joystickOptionsRender(struct JoystickOptions* joystickOptions, struct Rende
 
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_ENV_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
 
     gDPPipeSync(renderState->dl++);
     menuSetRenderColor(renderState, joystickOptions->selectedItem == JoystickOptionInvert, &gSelectionGray, &gColorWhite);
@@ -225,5 +225,5 @@ void joystickOptionsRender(struct JoystickOptions* joystickOptions, struct Rende
     menuSetRenderColor(renderState, joystickOptions->selectedItem == JoystickOptionDeadzone, &gSelectionGray, &gColorWhite);
     gSPDisplayList(renderState->dl++, joystickOptions->joystickDeadzoneText);
 
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 }

src/menu/landing_menu.c

@@ -110,11 +110,11 @@ void landingMenuRender(struct LandingMenu* landingMenu, struct RenderState* rend
     gSPDisplayList(renderState->dl++, portal_logo_gfx);
     gSPDisplayList(renderState->dl++, ui_material_revert_list[PORTAL_LOGO_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
     for (int i = 0; i < landingMenu->optionCount; ++i) {
         gDPPipeSync(renderState->dl++);
         menuSetRenderColor(renderState, landingMenu->selectedItem == i, &gSelectionGray, &gColorWhite);
         gSPDisplayList(renderState->dl++, landingMenu->optionText[i]);
     }
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 }

src/menu/new_game_menu.c

@@ -193,7 +193,7 @@ void newGameRender(struct NewGameMenu* newGameMenu, struct RenderState* renderSt
 
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_ENV_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
     gSPDisplayList(renderState->dl++, newGameMenu->newGameText);
 
     gDPPipeSync(renderState->dl++);
@@ -206,7 +206,7 @@ void newGameRender(struct NewGameMenu* newGameMenu, struct RenderState* renderSt
         menuSetRenderColor(renderState, newGameMenu->selectedChapter != newGameMenu->chapterOffset, &gSelectionOrange, &gColorWhite);
         gSPDisplayList(renderState->dl++, newGameMenu->chapter1.chapterText);
         gSPDisplayList(renderState->dl++, newGameMenu->chapter1.testChamberText);
-        gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+        gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
     }
 
     graphicsCopyImage(

src/menu/options_menu.c

@@ -119,9 +119,9 @@ void optionsMenuRender(struct OptionsMenu* options, struct RenderState* renderSt
     gSPDisplayList(renderState->dl++, options->tabs.tabOutline);
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_ENV_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
     renderState->dl = tabsRenderText(&options->tabs, renderState->dl);
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 
     switch (options->tabs.selectedTab) {
         case OptionsMenuTabsControlMapping:

src/menu/savefile_list.c

@@ -211,7 +211,7 @@ void savefileListRender(struct SavefileListMenu* savefileList, struct RenderStat
     }
     gSPDisplayList(renderState->dl++, ui_material_revert_list[SOLID_ENV_INDEX]);
 
-    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_list[DEJAVU_SANS_0_INDEX]);
 
     gDPPipeSync(renderState->dl++);
     gDPSetScissor(renderState->dl++, G_SC_NON_INTERLACE, 0, 0, SCREEN_WD, SCREEN_HT);
@@ -237,7 +237,7 @@ void savefileListRender(struct SavefileListMenu* savefileList, struct RenderStat
         renderStateInlineBranch(renderState, slot->gameId);
     }
 
-    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_INDEX]);
+    gSPDisplayList(renderState->dl++, ui_material_revert_list[DEJAVU_SANS_0_INDEX]);
 
     gSPDisplayList(renderState->dl++, ui_material_list[IMAGE_COPY_INDEX]);
 

tools/font_converter.js

@@ -1,26 +1,69 @@
 // this tool takes a the json output from https://github.com/andryblack/fontbuilder and create a font usabe in portal64
 
+// usage
+// generate multiple font files with each image not being larger than 4kb
+// then name the files font_file_0.json, font_file_1.json, font_file_2.json
+// you also need a json file with all the characters in a single file called font_file_all.json
+// this all file is only needed to extract the kerning
+// once you have that use this as follows
+//
+// node tools/font_converter.js FontName /path/to/font_file /path/to/output.c
+
 const fs = require('fs');
 
 const name = process.argv[2];
-const input = JSON.parse(fs.readFileSync(process.argv[3]));
+const filePrefix = process.argv[3];
 
-function hashFunction(first, second, multiplier, arraySize) {
+const joinedSymbols = [];
-    return ((first * multiplier) + second) % arraySize;
+let index = 0;
+
+while (index < 100) {
+    const filename = `${filePrefix}_${index}.json`;
+    if (!fs.existsSync(filename)) {
+        break;
     }
 
-function checkForCollisions(kerningList, multiplier, arraySize) {
+    const singleInput = JSON.parse(fs.readFileSync(filename));
+
+    singleInput.symbols.forEach((symbol) => {
+        joinedSymbols.push({
+            ...symbol,
+            textureIndex: index,
+        });
+    });
+
+    ++index;
+}
+
+const allSymbols = JSON.parse(fs.readFileSync(`${filePrefix}_all.json`));
+
+const input = {
+    kerning: allSymbols.kerning,
+    config: allSymbols.config,
+    symbols: joinedSymbols,
+};
+
+function kerningHashFunction(kerning, multiplier, arraySize) {
+    return ((kerning.first * multiplier) + kerning.second) % arraySize;
+}
+
+function symbolHashFunction(symbol, multiplier, arraySize) {
+    return (symbol.id * multiplier) % arraySize;
+}
+
+function checkForCollisions(list, hashFunction, multiplier, arraySize, emptyObj) {
     const sparseArray = [];
     sparseArray.length = arraySize;
 
-    if (arraySize < kerningList.length) {
+    if (arraySize < list.length) {
         return null;
     }
 
     let maxCollisions = 0;
+    let averageCollisions = 0;
 
-    for (const kerning of kerningList) {
+    for (const element of list) {
-        let index = hashFunction(kerning.first, kerning.second, multiplier, arraySize);
+        let index = hashFunction(element, multiplier, arraySize);
 
         let currentCollisions = 0;
 
@@ -31,34 +74,39 @@ function checkForCollisions(kerningList, multiplier, arraySize) {
 
         maxCollisions = Math.max(maxCollisions, currentCollisions);
 
-        sparseArray[index] = kerning;
+        averageCollisions += currentCollisions;
+
+        sparseArray[index] = element;
     }
 
+    averageCollisions /= list.length;
+
     for (let i = 0; i < arraySize; ++i) {
         if (!sparseArray[i]) {
-            sparseArray[i] = {amount: 0, first: 0, second: 0};
+            sparseArray[i] = emptyObj;
         }
     }
 
-    return {sparseArray, maxCollisions};
+    return {sparseArray, maxCollisions, averageCollisions};
 }
 
-function searchForBestKerning(kerningList) {
+function searchForBestHashTable(list, hashFunction, emptyObj) {
     let result;
     let multiplier;
 
     let arraySize = 1;
-    let mask = 2;
+    let mask = 1;
 
-    while (arraySize < kerningList.length) {
+    while (arraySize < list.length) {
         arraySize *= 2;
         mask <<= 1;
     }
 
     arraySize *= 2;
+    mask <<= 1;
 
     for (let i = 1; i < 0x10000; ++i) {
-        const check = checkForCollisions(kerningList, i, arraySize);
+        const check = checkForCollisions(list, hashFunction, i, arraySize, emptyObj);
 
         if (!result || check.maxCollisions < result.maxCollisions) {
             result = check
@@ -66,11 +114,9 @@ function searchForBestKerning(kerningList) {
         }
     }
 
-    console.log(`maxCollisions = ${result.maxCollisions}`);
-
     mask -= 1;
 
-    return {result: result.sparseArray, multiplier: multiplier, mask: mask, maxCollisions: result.maxCollisions};
+    return {result: result.sparseArray, multiplier: multiplier, mask: mask, maxCollisions: result.maxCollisions, averageCollisions: result.averageCollisions};
 }
 
 function buildKerning(kerningList) {
@@ -80,53 +126,51 @@ ${kerningList.map(kerning => `    {.amount = ${kerning.amount}, .first = ${kerni
 `
 }
 
-function buildFont(multiplier, mask, symbolCount, maxCollisions) {
+function buildFont(kerningResult, symbolResult) {
     return `struct Font g${name}Font = {
     .kerning = &g${name}Kerning[0],
     .symbols = &g${name}Symbols[0],
     .base = ${input.config.base},
     .charHeight = ${input.config.charHeight},
-    .symbolCount = ${symbolCount},
+    .symbolMultiplier = ${symbolResult.multiplier},
-    .kerningMultiplier = ${multiplier},
+    .symbolMask = 0x${symbolResult.mask.toString(16)},
-    .kerningMask = 0x${mask.toString(16)},
+    .symbolMaxCollisions = ${symbolResult.maxCollisions},
-    .maxCollisions = ${maxCollisions},
+    .kerningMultiplier = ${kerningResult.multiplier},
+    .kerningMask = 0x${kerningResult.mask.toString(16)},
+    .kerningMaxCollisions = ${kerningResult.maxCollisions},
 };
 `
 }
 
-function sparseSymbols(symbols) {
-    const result = [];
-
-    for (let i = 0; i < symbols.length; ++i) {
-        result[symbols[i].id] = symbols[i];
-    }
-
-    for (let i = 0; i < result.length; ++i) {
-        if (!result[i]) {
-            result[i] = {
-                x: 0, y: 0,
-                width: 0, height: 0,
-                xoffset: 0, yoffset: 0,
-                xadvance: 0,
-            };
-        }
-    }
-
-    return result;
-}
-
 function buildSymbol(symbol) {
     return `    {
+        .id = ${symbol.id},
         .x = ${symbol.x}, .y = ${symbol.y},
         .width = ${symbol.width}, .height = ${symbol.height},
         .xoffset = ${symbol.xoffset}, .yoffset = ${symbol.yoffset},
         .xadvance = ${symbol.xadvance},
+        .textureIndex = ${symbol.textureIndex},
     },`
 }
 
-const kerningResult = searchForBestKerning(input.kerning);
+const kerningResult = searchForBestHashTable(
+    input.kerning, 
+    kerningHashFunction, 
+    {amount: 0, first: 0, second: 0}
+);
 
-const symbols = sparseSymbols(input.symbols);
+const symbolResult = searchForBestHashTable(
+    input.symbols, 
+    symbolHashFunction, 
+    {id: 0, x: 0, y: 0, width: 0, height: 0, xoffset: 0, yoffset: 0, xadvance: 0, textureIndex: -1}
+);
+
+console.log(`symbolLength = ${input.symbols.length}/${symbolResult.result.length}`);
+console.log(`symbolMaxCollisions = ${symbolResult.maxCollisions}`);
+console.log(`symbolAverageCollisions = ${symbolResult.averageCollisions}`);
+console.log(`kerningLength = ${input.kerning.length}/${kerningResult.result.length}`);
+console.log(`maxKerningCollisions = ${kerningResult.maxCollisions}`);
+console.log(`kerningAverageCollisions = ${kerningResult.averageCollisions}`);
 
 fs.writeFileSync(process.argv[4], `
 
@@ -135,8 +179,8 @@ fs.writeFileSync(process.argv[4], `
 ${buildKerning(kerningResult.result)}
 
 struct FontSymbol g${name}Symbols[] = {
-${symbols.map(buildSymbol).join('\n')}
+${symbolResult.result.map(buildSymbol).join('\n')}
 };
 
-${buildFont(kerningResult.multiplier, kerningResult.mask, symbols.length, kerningResult.maxCollisions)}
+${buildFont(kerningResult, symbolResult)}
 `);

tools/level_scripts/subtitle_generate.py

@@ -2,6 +2,19 @@
 import os
 import re
 import sys
+import json
+
+def get_supported_characters():
+    with open('assets/fonts/dejavu_sans_book_8.json', 'r') as f:
+        content = json.loads('\n'.join(f.readlines()))
+
+    result = {' ', '\t', '\n', '\r'}
+    
+    for symbol in content['symbols']:
+        result.update(chr(symbol['id']))
+
+    return result
+
 
 def dump_lines(sourcefile_path, lines):
     if not os.path.exists(os.path.dirname(os.path.abspath(sourcefile_path))):
@@ -133,6 +146,21 @@ def make_subtitle_for_language(lang_lines, lang_name):
 
     dump_lines(f"build/src/audio/subtitles_{lang_name}.c", lines)
 
+def determine_invalid_characters(lang_name, lang_lines, good_characters):
+    used_characters = set()
+
+    for value in lang_lines:
+        used_characters = used_characters | set(value)
+
+    invalid = used_characters - good_characters
+
+    if len(invalid) == 0:
+        return
+    
+    print(f"{lang_name} has {len(invalid)} invalid charcters\n{''.join(sorted(list(invalid)))}")
+
+    return used_characters
+
 
 def make_subtitle_ld(languages):
     lines = []
@@ -189,11 +217,9 @@ def make_overall_subtitles_sourcefile(language_list):
 def process_all_closecaption_files(dir, language_names):
     values_list = []
     header_lines = []
-    sourcefile_lines = []
     language_list = []
     language_with_values_list = []
     SubtitleKey_generated = False
-    key_order = {}
 
     for langauge_name in language_names:
         filename = f"closecaption_{langauge_name}.txt"
@@ -202,7 +228,7 @@ def process_all_closecaption_files(dir, language_names):
             filepath = os.path.join(dir, filename)
             lines = []
             
-            with open(filepath, "r", encoding='cp1252') as f:
+            with open(filepath, "r", encoding='utf-16-le') as f:
                 lines = f.readlines()
             
             new_lines = []
@@ -223,12 +249,22 @@ def process_all_closecaption_files(dir, language_names):
                 'name': l,
             })
             print(filename, " - PASSED")
-        except:
+        except Exception as e:
+            print(e)
             print(filename, " - FAILED")
             continue
 
+    good_characters = get_supported_characters()
+    used_characters = set()
+
     for language in language_with_values_list:
         make_subtitle_for_language(language['value'], language['name'])
+        used_characters = used_characters | determine_invalid_characters(language['name'], language['value'], good_characters)
+
+    print(f"needed characters\n{''.join(sorted(list(good_characters & used_characters)))}")
+    print(f"unused characters\n{''.join(sorted(list(good_characters - used_characters)))}")
+    print(f"invalid characters\n{''.join(sorted(list(used_characters - good_characters)))}")
+
     make_subtitle_ld(language_with_values_list)
 
     make_overall_subtitles_header(header_lines, language_list)