gbdk-2020 | GameBoy Development Kit |
| download: https://git.y1.nz/archives/gbdk.tar.gz | |
| README | Files | Log | Refs | LICENSE |
commit 8873f5bc223f800cfa7c738e3e88aa9dd7fd5194 parent 6ff58a9ab7b72cbf409314cc0bf6464644891866 Author: bbbbbr <reg+github@roughhousing.com> Date: Thu, 23 Sep 2021 16:44:21 -0700 Merge pull request #270 from bbbbbr/gbcompress_work Gbcompress: fixes and improvements Diffstat:
| M | docs/pages/20_toolchain_settings.md | 15 | ++++++++++----- |
| M | gbdk-support/gbcompress/Makefile | 7 | ++++++- |
| M | gbdk-support/gbcompress/gbcompress.c | 39 | ++++++++++++++++++++++----------------- |
| M | gbdk-support/gbcompress/main.c | 53 | ++++++++++++++++++++++++++++++++++++++++------------- |
| A | gbdk-support/gbcompress/rlecompress.c | 230 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | gbdk-support/gbcompress/rlecompress.h | 12 | ++++++++++++ |
6 files changed, 320 insertions(+), 36 deletions(-)
diff --git a/docs/pages/20_toolchain_settings.md b/docs/pages/20_toolchain_settings.md @@ -10,7 +10,7 @@ -Bdir/ use the compiler named `dir/rcc' -c compile only -dn set switch statement density to `n' --debug turn on --debug for compiler, -y (.cdb) and -j (.noi) for linker +-debug Turns on --debug for compiler, -y (.cdb) and -j (.noi) for linker -Dname -Dname=def define the preprocessor symbol `name' -E run only the preprocessor on the named C programs and unsuffixed files -g produce symbol table information for debuggers @@ -422,17 +422,22 @@ Arguments: # gbcompress settings ``` gbcompress [options] infile outfile -Use: Gbcompress a binary file and write it out. +Use: compress a binary file and write it out. Options -h : Show this help screen -d : Decompress (default is compress) -v : Verbose output --cin : Read input as .c source format (8 bit char ONLY, uses first array found) --cout : Write output in .c / .h source format (8 bit char ONLY) --varname=<NAME> : specify variable name for c source output +--cin : Read input as .c source format (8 bit char ONLY, uses first array found) +--cout : Write output in .c / .h source format (8 bit char ONLY) +--varname=<NAME> : specify variable name for c source output +--alg=<type> : specify compression type. 'rle', 'gb' (default) Example: "gbcompress binaryfile.bin compressed.bin" Example: "gbcompress -d compressedfile.bin decompressed.bin" +Example: "gbcompress --alg=rle binaryfile.bin compressed.bin" + +The default compression (gb) is the type used by gbtd/gbmb +The rle compression is Amiga IFF style ``` @anchor png2asset-settings # png2asset settings diff --git a/gbdk-support/gbcompress/Makefile b/gbdk-support/gbcompress/Makefile @@ -6,7 +6,7 @@ endif CC = $(TOOLSPREFIX)gcc CFLAGS = -ggdb -O -Wno-incompatible-pointer-types -DGBDKLIBDIR=\"$(TARGETDIR)\" -OBJ = main.o gbcompress.o files.o files_c_source.o +OBJ = main.o gbcompress.o rlecompress.o files.o files_c_source.o BIN = gbcompress all: $(BIN) @@ -25,3 +25,8 @@ test: rm -f tmp.cmp.c; rm -f tmp.dcmp.c; rm -f tmp.cmp; rm -f tmp.dcmp cp $(BIN) tmp.in; ./gbcompress -v --cout --varname=some_array tmp.in tmp.cmp.c; ./gbcompress -v -d --cin tmp.cmp.c tmp.dcmp; diff -s tmp.in tmp.dcmp rm tmp.* + rm -f tmp.cmp; rm -f tmp.dcmp + cp $(BIN) tmp.in; ./gbcompress --alg=rle -v tmp.in tmp.cmp; ./gbcompress --alg=rle -v -d tmp.cmp tmp.dcmp; diff -s tmp.in tmp.dcmp + rm -f tmp.cmp.c; rm -f tmp.dcmp.c; rm -f tmp.cmp; rm -f tmp.dcmp + cp $(BIN) tmp.in; ./gbcompress --alg=rle -v --cout --varname=some_array tmp.in tmp.cmp.c; ./gbcompress --alg=rle -v -d --cin tmp.cmp.c tmp.dcmp; diff -s tmp.in tmp.dcmp + rm tmp.* diff --git a/gbdk-support/gbcompress/gbcompress.c b/gbdk-support/gbcompress/gbcompress.c @@ -76,7 +76,7 @@ static void write_string( uint8_t len, uint16_t data) { // conver's complement does not give the negation, see § Most negative number below. t back-ref offset from positive unsigned to negative signed data = (data ^ 0xFFFF) + 1; - + FoutBuf[FoutIndex++] = (((len - 1) & len_mask) | token_str); FoutBuf[FoutIndex++] = (uint8_t)(data & 0xFF); FoutBuf[FoutIndex++] = (uint8_t)((data >> 8) & 0xFF); @@ -103,7 +103,7 @@ static void write_end(void) { } -static int read_uint16_t(uint16_t byte_pos, uint16_t * out_data) { +static int read_uint16_t(uint32_t byte_pos, uint16_t * out_data) { if ((byte_pos + 2) < Fsize_in) { *out_data = (uint16_t)((FinBuf[byte_pos] << 8) + (uint16_t)FinBuf[byte_pos+1]); @@ -160,10 +160,10 @@ uint32_t gbcompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, if ((FinBuf[FinIndex + rle_u8_len] == rle_u8_match) && (rle_u8_len < 64)) { rle_u8_len++; - } + } else break; } - + // Check for overlapping uint16_t RLE run up to 63 bytes max // Read in initial u16 to match against if (read_uint16_t(FinIndex, &rle_u16_match)) { @@ -171,10 +171,10 @@ uint32_t gbcompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, rle_u16_len = 1; // If the current u16 matches, increment the length while (read_uint16_t(FinIndex + (rle_u16_len * 2), &temp_u16)) { - if ((temp_u16 == rle_u16_match) && + if ((temp_u16 == rle_u16_match) && (rle_u16_len < 64)) { rle_u16_len++; - } + } else break; } } @@ -182,13 +182,19 @@ uint32_t gbcompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, // Check for matching sequences starting at current position // against all previous data beginning at start up to 63 bytes max // (back reference "strings") - rle_str_start = 0; // rle_str_back_offset = 0; rle_str_len = 0; + // Max string backreference length is 16 bits unsigned + // adjust search start accordingly + if (FinIndex > 0xFFFF) + rle_str_start = FinIndex - 0xFFFF; + else + rle_str_start = 0; + while (rle_str_start < FinIndex) { rle_str_len_work = 0; - // Check for a matching run at rle_str_start against FinIndex + // Check for a matching run at rle_str_start against FinIndex // and save length to rle_str_len_work while ((FinIndex + rle_str_len_work) < Fsize_in) { // Test to see if u8 in current sequence matches the u8 for a previous sequence @@ -215,14 +221,14 @@ uint32_t gbcompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, // Write out any rle data if it's ready - if ((rle_u8_len > 2) && - (rle_u8_len > rle_u16_len) && + if ((rle_u8_len > 2) && + (rle_u8_len > rle_u16_len) && (rle_u8_len > rle_str_len)) { flush_trash(&FinIndex, &trash_len); write_byte(rle_u8_len, rle_u8_match); FinIndex = FinIndex + rle_u8_len; - } - else if ((rle_u16_len > 2) && + } + else if ((rle_u16_len > 2) && ((rle_u16_len*2) > rle_str_len)) { flush_trash(&FinIndex, &trash_len); write_word(rle_u16_len, rle_u16_match); @@ -236,7 +242,7 @@ uint32_t gbcompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, else if (trash_len >= 64) { write_trash(trash_len, &FinBuf[FinIndex-trash_len]); trash_len = 0; - } + } else { trash_len++; FinIndex++; @@ -327,20 +333,19 @@ uint32_t gbdecompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBu case token_str: // This token: copy N bytes from negative offset of current // DECODED memory location (back reference) - backref_index = (uint16_t)(read_single_byte()); - backref_index |= (uint16_t)((read_single_byte())<<8); + backref_index = ((uint16_t)read_single_byte() | (( (uint16_t)read_single_byte() ) << 8)); // Convert input from from Signed to Unsigned backref_index = (backref_index ^ 0xFFFF) + 1; // Assign the string back reference relative to the current buffer pointer - backref_index = FoutIndex - backref_index; + backref_index = (FoutIndex - (uint32_t)backref_index); break; case token_trash: // AKA "Trash Bytes" in GBTD // This token : copy next N bytes directly from ENCODED input break; } - + while (rle_len) { // Copy the decoded byte into VRAM diff --git a/gbdk-support/gbcompress/main.c b/gbdk-support/gbcompress/main.c @@ -9,21 +9,27 @@ #include <stdint.h> #include "gbcompress.h" +#include "rlecompress.h" #include "files.h" #include "files_c_source.h" #define MAX_STR_LEN 4096 +#define COMPRESSION_TYPE_GB 0 +#define COMPRESSION_TYPE_RLE_BLOCK 1 +#define COMPRESSION_TYPE_DEFAULT COMPRESSION_TYPE_GB + char filename_in[MAX_STR_LEN] = {'\0'}; char filename_out[MAX_STR_LEN] = {'\0'}; uint8_t * p_buf_in = NULL; uint8_t * p_buf_out = NULL; -bool opt_mode_compress = true; -bool opt_verbose = false; -bool opt_c_source_input = false; -bool opt_c_source_output = false; +bool opt_mode_compress = true; +bool opt_verbose = false; +bool opt_compression_type = COMPRESSION_TYPE_DEFAULT; +bool opt_c_source_input = false; +bool opt_c_source_output = false; char opt_c_source_output_varname[MAX_STR_LEN] = "var_name"; static void display_help(void); @@ -36,17 +42,22 @@ void cleanup(void); static void display_help(void) { fprintf(stdout, "gbcompress [options] infile outfile\n" - "Use: Gbcompress a binary file and write it out.\n" + "Use: compress a binary file and write it out.\n" "\n" "Options\n" "-h : Show this help screen\n" "-d : Decompress (default is compress)\n" "-v : Verbose output\n" - "-cin : Read input as .c source format (8 bit char ONLY, uses first array found)\n" - "-cout : Write output in .c / .h source format (8 bit char ONLY) \n" - "-varname=<NAME> : specify variable name for c source output\n" + "--cin : Read input as .c source format (8 bit char ONLY, uses first array found)\n" + "--cout : Write output in .c / .h source format (8 bit char ONLY) \n" + "--varname=<NAME> : specify variable name for c source output\n" + "--alg=<type> : specify compression type: 'rle', 'gb' (default)\n" "Example: \"gbcompress binaryfile.bin compressed.bin\"\n" "Example: \"gbcompress -d compressedfile.bin decompressed.bin\"\n" + "Example: \"gbcompress --alg=rle binaryfile.bin compressed.bin\"\n" + "\n" + "The default compression (gb) is the type used by gbtd/gbmb\n" + "The rle compression is Amiga IFF style\n" ); } @@ -76,6 +87,10 @@ int handle_args(int argc, char * argv[]) { opt_c_source_output = true; } else if (strstr(argv[i], "--varname=") == argv[i]) { snprintf(opt_c_source_output_varname, sizeof(opt_c_source_output_varname), "%s", argv[i] + 10); + } else if (strstr(argv[i], "--alg=gb") == argv[i]) { + opt_compression_type = COMPRESSION_TYPE_GB; + } else if (strstr(argv[i], "--alg=rle") == argv[i]) { + opt_compression_type = COMPRESSION_TYPE_RLE_BLOCK; } else if (strstr(argv[i], "-d") == argv[i]) { opt_mode_compress = false; } else @@ -129,14 +144,20 @@ static int compress() { p_buf_out = malloc(buf_size_out); if ((p_buf_in) && (p_buf_out) && (buf_size_in > 0)) { - out_len = gbcompress_buf(p_buf_in, buf_size_in, &p_buf_out, buf_size_out); + + if (opt_compression_type == COMPRESSION_TYPE_GB) + out_len = gbcompress_buf(p_buf_in, buf_size_in, &p_buf_out, buf_size_out); + else if (opt_compression_type == COMPRESSION_TYPE_RLE_BLOCK) + out_len = rlecompress_buf(p_buf_in, buf_size_in, &p_buf_out, buf_size_out); + else + return EXIT_FAILURE; if (out_len > 0) { if (opt_c_source_output) { c_source_set_sizes(out_len, buf_size_in); // compressed, decompressed result = file_write_c_output_from_buffer(filename_out, p_buf_out, out_len, opt_c_source_output_varname, true); - } + } else result = file_write_from_buffer(filename_out, p_buf_out, out_len); @@ -148,7 +169,7 @@ static int compress() { } } - return EXIT_FAILURE; + return EXIT_FAILURE; } @@ -170,8 +191,14 @@ static int decompress() { p_buf_out = malloc(buf_size_out); if ((p_buf_in) && (p_buf_out) && (buf_size_in > 0)) { - out_len = gbdecompress_buf(p_buf_in, buf_size_in, - &p_buf_out, buf_size_out); + + if (opt_compression_type == COMPRESSION_TYPE_GB) + out_len = gbdecompress_buf(p_buf_in, buf_size_in, &p_buf_out, buf_size_out); + else if (opt_compression_type == COMPRESSION_TYPE_RLE_BLOCK) + out_len = rledecompress_buf(p_buf_in, buf_size_in, &p_buf_out, buf_size_out); + else + return EXIT_FAILURE; + if (out_len > 0) { if (opt_c_source_output) { diff --git a/gbdk-support/gbcompress/rlecompress.c b/gbdk-support/gbcompress/rlecompress.c @@ -0,0 +1,230 @@ +// This is free and unencumbered software released into the public domain. +// For more information, please refer to <https://unlicense.org> +// bbbbbr 2020 + +#include <stdio.h> +//#include <string.h> +#include <stdlib.h> +#include <stdbool.h> +#include <stdint.h> +#include "rlecompress.h" + + +static uint8_t * FinBuf = NULL; +static uint32_t Fsize_in = 0; +static uint32_t FinIndex = 0; + +static uint8_t ** pp_FoutBuf = NULL; +static uint8_t * FoutBuf = NULL; +static uint32_t Fsize_out = 0; +static uint32_t FoutIndex = 0; + + +#define RLE_MASK_LEN 0x7F +#define RLE_MASK_TYPE 0x80 +#define RLE_TYPE_RAND 0x00 +#define RLE_TYPE_REPEAT 0x80 +#define RLE_MAX_LEN 127 +#define RLE_CTRL_END 0x00 +#define RLE_MAX_BLOCK 127 +#define RLE_CHANGE_COST 2 + + +static uint8_t rle_queued[128]; +static int rle_queue_idx = 0; +static int run_len = 0; + + +// Initialize the buffer vars +static void initbufs(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, uint32_t size_out) { + + FinBuf = inBuf; + Fsize_in = size_in; + FinIndex = 0; + + pp_FoutBuf = pp_outBuf; + FoutBuf = *pp_outBuf; + Fsize_out = size_out; + FoutIndex = 0; +} + + +static void check_write_size(int len) { + + // Grow output buffer if needed + if ((FoutIndex + len) >= Fsize_out) { + uint8_t * p_tmp = *pp_FoutBuf; + + // Reallocate to twice as large + Fsize_out = Fsize_out * 2; + *pp_FoutBuf = (void *)realloc(*pp_FoutBuf, Fsize_out); + + // If realloc failed, free original buffer before quitting + if (!(*pp_FoutBuf)) { + printf("Error: Failed to grow memory for output buffer!\n"); + if (p_tmp) free(p_tmp); + p_tmp = NULL; + exit(EXIT_FAILURE); + } else + FoutBuf = *pp_FoutBuf; // Update working pointer + } +} + + + +static void write_end_of_data(void) { + + check_write_size(1); // writing 1 control byte + FoutBuf[FoutIndex++] = RLE_CTRL_END; // Write sequence control byte +} + + +static void write_run_repeat(int len, uint8_t value) { + + if (len) { + check_write_size(2); // writing 1 repeated value bytes + 1 control byte + // Write sequence control byte (length + type) + // Convert length to twos complement to identify it as repeat run + FoutBuf[FoutIndex++] = ((len & RLE_MASK_LEN) ^ 0xFFu) + 1u; + FoutBuf[FoutIndex++] = value; + } +} + + +static void rle_commit() { + + int idx = 0; + + if (rle_queue_idx > 0) { + check_write_size(rle_queue_idx + 1); // writing len bytes + 1 control byte + // Write sequence control byte (length + type) + FoutBuf[FoutIndex++] = (rle_queue_idx & RLE_MASK_LEN) | RLE_TYPE_RAND; + while (rle_queue_idx > 0) { + FoutBuf[FoutIndex++] = rle_queued[idx++]; + rle_queue_idx--; + } + } + rle_queue_idx = 0; // redundant +} + + + +// Convert buffer inBuf to rlecompress rle encoding and write out to outBuf +// Returns converted length +uint32_t rlecompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, uint32_t size_out) { + + uint8_t last, current; + initbufs(inBuf, size_in, pp_outBuf, size_out); + + last = 0; + run_len = 0; + rle_queue_idx = 0; + + while (FinIndex < Fsize_in) { + + current = FinBuf[FinIndex++]; + + if (current != last) { + // The run stopped, if switching to repeat is worthwhile then + // flush preceding random data + if (run_len > RLE_CHANGE_COST) { + rle_commit(); + write_run_repeat(run_len, last); + } else { + // Otherwise treat repeat data as random and queue it + // with flushing as needed + while(run_len--) { + if (rle_queue_idx >= RLE_MAX_BLOCK) { + rle_commit(); + } + rle_queued[rle_queue_idx++] = last; + } + } + run_len = 1; + last = current; + } + else { + // Flush pending repeat if max length is encountered + if (run_len >= RLE_MAX_BLOCK) { + rle_commit(); + write_run_repeat(run_len, last); + run_len = 0; + } + run_len++; + } + + } + + // If switching to repeat is worthwhile then flush preceding random data + if (run_len > RLE_CHANGE_COST) { + rle_commit(); + write_run_repeat(run_len, last); + } else { + // Otherwise treat repeat data as random and queue it + // with flushing as needed + while(run_len--) { + if (rle_queue_idx >= RLE_MAX_BLOCK) { + rle_commit(); + } + rle_queued[rle_queue_idx++] = last; + } + } + + write_end_of_data(); + + return FoutIndex; + +} + + + +static void write_single_byte(uint8_t data) { + + check_write_size(1); + + FoutBuf[FoutIndex++] = data; +} + + +static uint8_t read_single_byte(void) { + + if (FinIndex >= Fsize_in) { + printf("Error: Read past end of input buffer!\n"); + exit(EXIT_FAILURE); + } + + return (FinBuf[FinIndex++]); +} + + + +// Decompress buffer inBuf from gbcompress rle encoding and write to outBuf +// Returns converted length +uint32_t rledecompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, uint32_t size_out) { + + initbufs(inBuf, size_in, pp_outBuf, size_out); + + uint8_t token, rle_len, value; + + while (FinIndex < size_in) { + + token = read_single_byte(); + + // Check for EOF token, exit if encountered + if (token == RLE_CTRL_END) + break; + else if ((token & RLE_MASK_TYPE) == RLE_TYPE_RAND) { + rle_len = token & RLE_MASK_LEN; + while (rle_len--) + write_single_byte(read_single_byte()); + } + else { // if ((token & RLE_MASK_TYPE) == RLE_TYPE_REPEAT) { + rle_len = ((token ^ 0xFF) + 1) & RLE_MASK_LEN; // length is two's complement + value = read_single_byte(); + while (rle_len--) + write_single_byte(value); + } + } + + return FoutIndex; +} diff --git a/gbdk-support/gbcompress/rlecompress.h b/gbdk-support/gbcompress/rlecompress.h @@ -0,0 +1,12 @@ +// This is free and unencumbered software released into the public domain. +// For more information, please refer to <https://unlicense.org> +// bbbbbr 2020 + +#ifndef _RLECOMPRESS_H +#define _RLECOMPRESS_H + +uint32_t rlecompress_buf(uint8_t * inBuf, uint32_t InSize, uint8_t ** pp_outBuf, uint32_t size_out); +uint32_t rledecompress_buf(uint8_t * inBuf, uint32_t size_in, uint8_t ** pp_outBuf, uint32_t size_out); +uint32_t rlecompress_buf_escaped(uint8_t * inBuf, uint32_t InSize, uint8_t ** pp_outBuf, uint32_t size_out); + +#endif // _RLECOMPRESS_H
This webpage is intended to be an accessible preview of this repository. To get a fuller picture, clone it and use the git CLI.