locale parsing done

src/dusk/asset/assetfile.c

@@ -36,6 +36,19 @@ errorret_t assetFileInit(
   errorOk();
 }
 
+errorret_t assetFileRewind(assetfile_t *file) {
+  assertNotNull(file, "Asset file cannot be NULL.");
+  assertNotNull(file->zipFile, "Asset file must be opened before rewinding.");
+
+  if(file->position == 0) {
+    errorOk();
+  }
+
+  errorChain(assetFileClose(file));
+  errorChain(assetFileOpen(file));
+  errorOk();
+}
+
 errorret_t assetFileOpen(assetfile_t *file) {
   assertNotNull(file, "Asset file cannot be NULL.");
   assertNotNull(file->filename, "Asset file filename cannot be NULL.");
@@ -86,4 +99,211 @@ errorret_t assetFileDispose(assetfile_t *file) {
   }
   memoryZero(file, sizeof(assetfile_t));
   errorOk();
+}
+
+// Line Reader;
+void assetFileLineReaderInit(
+  assetfilelinereader_t *reader,
+  assetfile_t *file,
+  uint8_t *readBuffer,
+  const size_t readBufferSize,
+  uint8_t *outBuffer,
+  const size_t outBufferSize
+) {
+  assertNotNull(reader, "Line reader cannot be NULL.");
+  assertNotNull(file, " File cannot be NULL.");
+  assertNotNull(readBuffer, "Read buffer cannot be NULL.");
+  assertNotNull(outBuffer, "Output buffer cannot be NULL.");
+  assertTrue(readBufferSize > 0, "Read buffer size must be greater than 0.");
+  assertTrue(outBufferSize > 0, "Output buffer size must be greater than 0.");
+  
+  memoryZero(reader, sizeof(assetfilelinereader_t));
+
+  reader->file = file;
+  reader->readBuffer = readBuffer;
+  reader->readBufferSize = readBufferSize;
+  reader->outBuffer = outBuffer;
+  reader->outBufferSize = outBufferSize;
+}
+
+size_t assetFileLineReaderUnreadBytes(const assetfilelinereader_t *reader) {
+  assertNotNull(reader, "Reader cannot be NULL.");
+  assertTrue(reader->bufferEnd >= reader->bufferStart, "Invalid buffer state.");
+  return reader->bufferEnd - reader->bufferStart;
+}
+
+const uint8_t *assetFileLineReaderUnreadPtr(const assetfilelinereader_t *reader) {
+  assertNotNull(reader, "Reader cannot be NULL.");
+  assertNotNull(reader->readBuffer, "Read buffer cannot be NULL.");
+  return reader->readBuffer + reader->bufferStart;
+}
+
+static errorret_t assetFileLineReaderAppend(
+  assetfilelinereader_t *reader,
+  const uint8_t *src,
+  size_t srcLength
+) {
+  assertNotNull(reader, "Reader cannot be NULL.");
+  assertNotNull(reader->outBuffer, "Out buffer cannot be NULL.");
+
+  if(srcLength == 0) {
+    errorOk();
+  }
+
+  /* reserve room for optional NUL terminator */
+  if (reader->lineLength + srcLength >= reader->outBufferSize) {
+    errorThrow("Line length exceeds output buffer size.");
+  }
+
+  memoryCopy(reader->outBuffer + reader->lineLength, src, srcLength);
+  reader->lineLength += srcLength;
+  errorOk();
+}
+
+static void assetFileLineReaderTerminate(assetfilelinereader_t *reader) {
+  assertNotNull(reader, "Reader cannot be NULL.");
+  assertNotNull(reader->outBuffer, "Out buffer cannot be NULL.");
+  assertTrue(reader->lineLength < reader->outBufferSize, "Line length exceeds out buffer.");
+  reader->outBuffer[reader->lineLength] = '\0';
+}
+
+static ssize_t assetFileLineReaderFindNewline(const assetfilelinereader_t *reader) {
+  size_t i;
+
+  assertNotNull(reader, "Reader cannot be NULL.");
+  assertNotNull(reader->readBuffer, "Read buffer cannot be NULL.");
+
+  for (i = reader->bufferStart; i < reader->bufferEnd; ++i) {
+    if (reader->readBuffer[i] == '\n') {
+      return (ssize_t)i;
+    }
+  }
+
+  return -1;
+}
+
+errorret_t assetFileLineReaderFill(assetfilelinereader_t *reader) {
+
+  assertNotNull(reader, "Reader cannot be NULL.");
+  assertNotNull(reader->file, "File cannot be NULL.");
+  assertNotNull(reader->readBuffer, "Read buffer cannot be NULL.");
+
+  if(reader->eof) errorOk();
+
+  errorret_t ret;
+
+  size_t unreadBytes = assetFileLineReaderUnreadBytes(reader);
+
+  /* If buffer is fully consumed, refill from start. */
+  if (unreadBytes == 0) {
+    reader->bufferStart = 0;
+    reader->bufferEnd = 0;
+
+    errorChain(assetFileRead(
+      reader->file,
+      reader->readBuffer,
+      reader->readBufferSize
+    ));
+
+    if(reader->file->lastRead == 0) {
+      reader->eof = true;
+      errorOk();
+    }
+
+    reader->bufferStart = 0;
+    reader->bufferEnd = reader->file->lastRead;
+    errorOk();
+  }
+
+  /*
+   * There are unread bytes left but no newline in them.
+   * If bufferStart > 0, slide unread bytes to front so we can read more.
+   * This only happens when necessary to make space.
+   */
+  if(reader->bufferEnd == reader->readBufferSize) {
+    if(reader->bufferStart == 0) {
+      /*
+       * Entire read buffer is unread and contains no newline.
+       * Caller must have a large enough outBuffer to accumulate across fills,
+       * so we consume these bytes into outBuffer before refilling.
+       */
+      errorOk();
+    }
+
+    memoryMove(
+      reader->readBuffer,
+      reader->readBuffer + reader->bufferStart,
+      unreadBytes
+    );
+    reader->bufferStart = 0;
+    reader->bufferEnd = unreadBytes;
+  }
+
+  errorChain(assetFileRead(
+    reader->file,
+    reader->readBuffer + reader->bufferEnd,
+    reader->readBufferSize - reader->bufferEnd
+  ));
+
+  if(reader->file->lastRead == 0) {
+    reader->eof = true;
+    errorOk();
+  }
+
+  reader->bufferEnd += reader->file->lastRead;
+  errorOk();
+}
+
+errorret_t assetFileLineReaderNext(assetfilelinereader_t *reader) {
+  assertNotNull(reader, "Reader cannot be NULL.");
+  assertNotNull(reader->file, "File cannot be NULL.");
+  assertNotNull(reader->readBuffer, "Read buffer cannot be NULL.");
+  assertNotNull(reader->outBuffer, "Out buffer cannot be NULL.");
+
+  reader->lineLength = 0;
+
+  for (;;) {
+    ssize_t newlineIndex = assetFileLineReaderFindNewline(reader);
+
+    if (newlineIndex >= 0) {
+      size_t chunkLength = (size_t)newlineIndex - reader->bufferStart;
+      errorret_t ret;
+
+      /* strip CR in CRLF */
+      if (chunkLength > 0 && reader->readBuffer[(size_t)newlineIndex - 1] == '\r') {
+        chunkLength--;
+      }
+
+      errorChain(assetFileLineReaderAppend(
+        reader,
+        reader->readBuffer + reader->bufferStart,
+        chunkLength
+      ));
+      
+      reader->bufferStart = (size_t)newlineIndex + 1;
+      assetFileLineReaderTerminate(reader);
+      errorOk();
+    }
+
+    if (assetFileLineReaderUnreadBytes(reader) > 0) {
+      errorChain(assetFileLineReaderAppend(
+        reader,
+        assetFileLineReaderUnreadPtr(reader),
+        assetFileLineReaderUnreadBytes(reader)
+      ));
+
+      reader->bufferStart = reader->bufferEnd;
+    }
+
+    if(reader->eof) {
+      if(reader->lineLength > 0) {
+        assetFileLineReaderTerminate(reader);
+        errorOk();
+      }
+
+      errorThrow("End of file reached.");
+    }
+
+    errorChain(assetFileLineReaderFill(reader));
+  }
 }

src/dusk/asset/assetfile.h

@@ -51,6 +51,13 @@ errorret_t assetFileInit(
  */
 errorret_t assetFileOpen(assetfile_t *file);
 
+/**
+ * Rewind the file to the initial position.
+ * 
+ * @param file The asset file to rewind.
+ */
+errorret_t assetFileRewind(assetfile_t *file);
+
 /**
  * Read bytes from the asset file. Assumes the file has already been opened
  * prior to trying to read anything.
@@ -80,4 +87,52 @@ errorret_t assetFileClose(assetfile_t *file);
  * @param file The asset file to dispose.
  * @return An error code if the file could not be disposed properly.
  */
-errorret_t assetFileDispose(assetfile_t *file);
+errorret_t assetFileDispose(assetfile_t *file);
+
+typedef struct {
+  assetfile_t *file;
+  uint8_t *readBuffer;
+  size_t readBufferSize;
+  uint8_t *outBuffer;
+  size_t outBufferSize;
+
+  // A
+  size_t bufferStart;
+  size_t bufferEnd;
+  bool_t eof;//?
+
+  // Updated each reach:
+  size_t lineLength;
+} assetfilelinereader_t;
+
+/**
+ * Initializes a line reader for the given asset file. The line reader will read
+ * lines from the file into the provided line buffer, using the provided buffer
+ * for reading chunks of the file.
+ * 
+ * @param file The asset file to read from. Must already be opened.
+ * @param readBuffer Buffer to use for reading chunks of the file.
+ * @param readBufferSize Size of the read buffer.
+ * @param outBuffer Buffer to read lines into. Lines will be null-terminated.
+ * @param outBufferSize Size of the output buffer.
+ * @return An initialized line reader structure.
+ */
+void assetFileLineReaderInit(
+  assetfilelinereader_t *reader,
+  assetfile_t *file,
+  uint8_t *readBuffer,
+  const size_t readBufferSize,
+  uint8_t *outBuffer,
+  const size_t outBufferSize
+);
+
+/**
+ * Reads the next line from the asset file into the line buffer. The line
+ * buffer is null-terminated and does not include the newline character.
+ * 
+ * @param reader The line reader to read from.
+ * @return An error code if a failure occurs, or errorOk() if a line was read
+ *         successfully. If the end of the file is reached, errorEndOfFile() is
+ *         returned.
+ */
+errorret_t assetFileLineReaderNext(assetfilelinereader_t *reader);

src/dusk/asset/loader/locale/assetlocaleloader.c

@@ -6,6 +6,10 @@
  */
 
 #include "assetlocaleloader.h"
+#include "util/memory.h"
+#include "util/math.h"
+#include "util/string.h"
+#include "assert/assert.h"
 
 errorret_t assetLocaleLoader(assetfile_t *file) {
   errorThrow("Locale asset loading is not yet implemented.");
@@ -16,4 +20,259 @@ errorret_t assetLocaleLoad(
   void *nothing
 ) {
   errorThrow("Locale asset loading is not yet implemented.");
+}
+
+errorret_t assetLocaleLineSkipBlanks(
+  assetfilelinereader_t *reader,
+  uint8_t *lineBuffer
+) {
+  while(!reader->eof) {
+    // Skip blank lines
+    if(lineBuffer[0] == '\0') {
+      errorChain(assetFileLineReaderNext(reader));
+      continue;
+    }
+
+    // Skip comment lines
+    if(lineBuffer[0] == '#') {
+      errorChain(assetFileLineReaderNext(reader));
+      continue;
+    }
+    
+    // Is line only spaces?
+    size_t lineLength = strlen((char_t *)lineBuffer);
+    size_t i;
+    bool_t onlySpaces = true;
+    for(i = 0; i < lineLength; i++) {
+      if(lineBuffer[i] != ' ') {
+        onlySpaces = false;
+        break;
+      }
+    }
+
+    if(onlySpaces) {
+      errorChain(assetFileLineReaderNext(reader));
+      continue;
+    }
+    break;
+  }
+
+  errorOk();
+}
+
+errorret_t assetLocaleLineUnbuffer(
+  assetfilelinereader_t *reader,
+  uint8_t *lineBuffer,
+  uint8_t *stringBuffer,
+  const size_t stringBufferSize
+) {
+  stringBuffer[0] = '\0';
+
+  // At the point this funciton is called, we are looking for an opening quote.
+  char_t *start = strchr((char_t *)lineBuffer, '"');
+  if(!start) {
+    errorThrow("Expected open (0) \"");
+  }
+
+  char *end = strchr(start + 1, '"');
+  if(!end) {
+    errorThrow("Expected close (0) \"");
+  }
+  *end = '\0';
+
+  if(strlen(start) >= stringBufferSize) {
+    errorThrow("String buffer overflow");
+  }
+  memoryCopy(stringBuffer, start + 1, strlen(start));
+
+  // Now start buffering lines out
+  while(!reader->eof) {
+    errorChain(assetFileLineReaderNext(reader));
+
+    // Skip blank lines
+    errorChain(assetLocaleLineSkipBlanks(reader, lineBuffer));
+    
+    // Skip starting spaces
+    char_t *ptr = (char_t *)lineBuffer;
+    while(*ptr == ' ') {
+      ptr++;
+    }
+
+    // Only consider lines starting with quote
+    if(*ptr != '"') {
+      break;
+    }
+
+    ptr++; // move past first quote
+    
+    bool_t escaping = false;
+    char_t *dest = (char_t *)stringBuffer + strlen((char_t *)stringBuffer);
+    while(*ptr) {
+      if(escaping) {
+        // Handle escape sequences
+        switch(*ptr) {
+          case 'n': *dest++ = '\n'; break;
+          case 't': *dest++ = '\t'; break;
+          case '\\': *dest++ = '\\'; break;
+          case '"': *dest++ = '"'; break;
+          default:
+            errorThrow("Unknown escape sequence: \\%c", *ptr);
+        }
+        escaping = false;
+      } else if(*ptr == '\\') {
+        escaping = true;
+      } else if(*ptr == '"') {
+        // End of string
+        break;
+      } else {
+        // Regular character
+        *dest++ = *ptr;
+      }
+      if((size_t)(dest - (char_t *)stringBuffer) >= stringBufferSize) {
+        errorThrow("String buffer overflow");
+      }
+      ptr++;
+    }
+    *dest = '\0';
+  }
+
+  errorOk();
+}
+
+errorret_t assetLocaleGetString(
+  assetfile_t *file,
+  const char_t *messageId,
+  const int32_t pluralIndex,
+  char_t *stringBuffer,
+  const size_t stringBufferSize
+) {
+  assertNotNull(file, "Asset file cannot be NULL.");
+  assertNotNull(messageId, "Message ID cannot be NULL.");
+  assertTrue(pluralIndex >= 0, "Plural index cannot be negative.");
+  assertNotNull(stringBuffer, "String buffer cannot be NULL.");
+  assertTrue(stringBufferSize > 0, "String buffer size must be > 0");
+  assetfilelinereader_t reader;
+
+  bool_t msgidFound = false, msgidPluralFound = false, msgstrFound = false;
+  uint8_t msgidBuffer[256];
+  uint8_t msgidPluralBuffer[256];
+  uint8_t readBuffer[1024];
+  uint8_t lineBuffer[1024];
+
+  msgidBuffer[0] = '\0';
+  msgidPluralBuffer[0] = '\0';
+  stringBuffer[0] = '\0';
+
+  // Rewind and start reading lines.
+  errorChain(assetFileRewind(&LOCALE.file));
+  assetFileLineReaderInit(
+    &reader,
+    &LOCALE.file,
+    readBuffer,
+    sizeof(readBuffer),
+    lineBuffer,
+    sizeof(lineBuffer)
+  );
+
+  // Skip blanks, comments, etc and start looking for msgid's
+  errorChain(assetLocaleLineSkipBlanks(&reader, lineBuffer));
+
+  while(!reader.eof) {
+    // Is this msgid?
+    if(memoryCompare(lineBuffer, "msgid", 5) != 0) {
+      errorChain(assetFileLineReaderNext(&reader));
+      msgidBuffer[0] = '\0';
+      continue;
+    }
+
+    // Unbuffer the msgid
+    assetLocaleLineUnbuffer(
+      &reader, lineBuffer, (uint8_t *)msgidBuffer, sizeof(msgidBuffer)
+    );
+
+    // Is this the needle?
+    if(memoryCompare(msgidBuffer, messageId, strlen(messageId)) != 0) {
+      continue;
+    }
+
+    msgidFound = true;
+    break;
+  }
+  if(!msgidFound) {
+    errorThrow("Failed to find message ID: %s", messageId);
+  }
+  
+  // We are either going to see a msgstr or a msgid_plural
+  while(!reader.eof) {
+    errorChain(assetLocaleLineSkipBlanks(&reader, lineBuffer));
+
+    // Is msgid_plural?
+    if(
+      !msgidPluralFound &&
+      memoryCompare(lineBuffer, "msgid_plural", 12) == 0
+    ) {
+      // Yes, start reading plural ID
+      assetLocaleLineUnbuffer(
+        &reader,
+        lineBuffer,
+        (uint8_t *)msgidPluralBuffer,
+        sizeof(msgidPluralBuffer)
+      );
+      msgidPluralFound = true;
+      printf("Found plural ID: %s\n", msgidPluralBuffer);
+      continue;
+    }
+
+    // Should be msgstr if not plural.
+    if(memoryCompare(lineBuffer, "msgstr", 6) != 0) {
+      errorThrow("Expected msgstr after msgid, found: %s", lineBuffer);
+      continue;
+    }
+
+    // If plural we need an index
+    if(msgidPluralFound) {
+      // Skip blank chars
+      char_t *ptr = (char_t *)lineBuffer + 6;
+      while(*ptr == ' ') {
+        ptr++;
+      }
+
+      if(*ptr != '[') {
+        errorThrow("Expected [ for plural form, found: %s", lineBuffer);
+      }
+
+      ptr++; // move past [
+
+      // Parse until ]
+      char *end = strchr(ptr, ']');
+      if(!end) {
+        errorThrow("Expected ] for plural form, found: %s", lineBuffer);
+      }
+      *end = '\0';
+
+      int32_t index = atoi(ptr);
+      if(index != pluralIndex) {
+        // Not the plural form we want, skip
+        errorChain(assetFileLineReaderNext(&reader));
+        continue;
+      }
+
+      // Undo damage to line buffer from unbuffering.
+      *end = ']';
+    }
+
+    // Parse out msgstr
+    errorChain(assetLocaleLineUnbuffer(
+      &reader, lineBuffer, (uint8_t *)stringBuffer, stringBufferSize
+    ));
+    msgstrFound = true;
+    break;
+  };
+
+  if(!msgstrFound) {
+    errorThrow("Failed to find msgstr for message ID: %s", messageId);
+  }
+
+  printf("Found translation: %s\n", stringBuffer);
+  errorOk();
 }

src/dusk/asset/loader/locale/assetlocaleloader.h

@@ -9,10 +9,6 @@
 #include "asset/asset.h"
 #include "locale/localemanager.h"
 
-typedef struct {
-  void *nothing;
-} assetlocaleloaderparams_t;
-
 /**
  * Handler for locale assets.
  * 
@@ -31,4 +27,53 @@ errorret_t assetLocaleLoader(assetfile_t *file);
 errorret_t assetLocaleLoad(
   const char_t *path,
   void *nothing
+);
+
+/**
+ * Skips blank lines and comment lines in the line reader.
+ * 
+ * @param reader Line reader to read from.
+ * @param lineBuffer Buffer to use for reading lines.
+ * @return Any error that occurs during skipping.
+ */
+errorret_t assetLocaleLineSkipBlanks(
+  assetfilelinereader_t *reader,
+  uint8_t *lineBuffer
+);
+
+/**
+ * Unbuffers a potentially multi-line quoted string from the line reader.
+ * 
+ * This will read lines until it finds a line that starts with a quote, then
+ * read until the closing quote.
+ * 
+ * @param reader Line reader to read from.
+ * @param lineBuffer Buffer to use for reading lines.
+ * @param stringBuffer Buffer to write the unbuffered string to.
+ * @param stringBufferSize Size of the string buffer.
+ * @return Any error that occurs during unbuffering.
+ */
+errorret_t assetLocaleLineUnbuffer(
+  assetfilelinereader_t *reader,
+  uint8_t *lineBuffer,
+  uint8_t *stringBuffer,
+  const size_t stringBufferSize
+);
+
+/**
+ * Test function for locale asset loading.
+ * 
+ * @param file Asset file to test loading from.
+ * @param messageId The message ID to retrieve.
+ * @param pluralIndex The plural index to retrieve.
+ * @param stringBuffer Buffer to write the retrieved string to.
+ * @param stringBufferSize Size of the string buffer.
+ * @return Any error that occurs during testing.
+ */
+errorret_t assetLocaleGetString(
+  assetfile_t *file,
+  const char_t *messageId,
+  const int32_t pluralIndex,
+  char_t *stringBuffer,
+  const size_t stringBufferSize
 );