From 92c38a2a4fc9fc5b9eae6e79b2ac531155cd8d98 Mon Sep 17 00:00:00 2001 From: cypress Date: Sat, 6 Jan 2024 15:32:18 -0500 Subject: [PATCH] Initial Commit --- LICENSE | 121 +++++++++++++++++++++++++++++ README.md | 40 ++++++++++ qc_hash_generator.py | 178 +++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 + 4 files changed, 342 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 qc_hash_generator.py create mode 100644 requirements.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1db0f69 --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +# (FTE-)QuakeC Hash Table Generator + +`QCHashTableGenerator` is a simple utility designed to take in `.CSV` input files and generate an FTEQCC-compilable hash table structure, hashed using XMODEM CRC16. The overall goal is to ease compute time by mitigating the overall amount of string comparisons your QuakeC project may perform. + +It makes the assumption that the first value in your input `.CSV` is the value you want to hash. Given this example, `old_path` will be replaced with a generated hash: + +```csv +old_path,current_path +progs/player.mdl,models/player.mdl +``` + +Producing output similar to this: + +```c +var struct { + float old_path_crc; + string current_path; + float crc_strlen; +} struct_name[] = { + {63799, "models/player.mdl", 16} // progs/player.mdl +} +``` + +The output is sorted in ascending order, by the hashed strings, so it will be easy to use in a binary search. + +The purpose of the `crc_strlen` field is for the develoepr to, if they deem necessary, perform a `strlen` check on their input string and compare it with the provided `crc_strlen` in an effort to detect hash collisions. + +## Installation & Usage + +`QCHashTableGenerator` is designed to support python 3.7 and above. The following modules are required to run the generator: + +```c +colorama==0.4.6 +fastcrc==0.3.0 +pandas==2.1.4 +``` + +They can be installed automatically with `pip -r requirements.txt`. + +`qc_hash_generator.py --help` will give you a list of arguments to use to generate your QuakeC source file. But to just get off the ground, `qc_hash_generator.py -i some.csv` is all you will need. \ No newline at end of file diff --git a/qc_hash_generator.py b/qc_hash_generator.py new file mode 100644 index 0000000..516e645 --- /dev/null +++ b/qc_hash_generator.py @@ -0,0 +1,178 @@ +""" +Nazi Zombies: Portable QuakeC CRC generator + +Takes input .CSV files and outputs an FTEQCC-compilable +QuakeC struct with its contents, always assumes the first +entry should be XMODEM CRC16 hashed, adding its length +as an entry as well, for collision detection. +""" + +import argparse +import pandas +import sys +import os +from fastcrc import crc16 +from colorama import Fore, Style +from dataclasses import dataclass + +args = {} +struct_fields = [] +original_lengths = [] +original_names = [] + +COL_BLUE = Fore.BLUE +COL_RED = Fore.RED +COL_YEL = Fore.YELLOW +COL_GREEN = Fore.GREEN +COL_NONE = Style.RESET_ALL + +ITYPE_FLOAT = 0 +ITYPE_STRING = 1 +ITYPE_CRC = 2 + +@dataclass +class StructField: + ''' + Class for fields that are added to the QuakeC struct. + ''' + name: str + item_type: int = ITYPE_FLOAT + +def write_qc_file(csv_data): + ''' + Writes the data obtained into an FTEQCC-compilable + struct. + ''' + with open(args['output_file'], 'w') as output: + # Define the struct. + output.write('var struct {\n') + + # Write out all of it's types.. + for fields in struct_fields: + if fields.item_type == ITYPE_STRING: + output.write('string ') + else: + output.write('float ') + + output.write(f'{fields.name};\n') + + # Close the struct. + output.write('}') + + # Now, the name of it + struct_name = args['struct_name'] + output.write(f'{struct_name}[]=') + output.write('{\n') + + # We can begin writing the actual data.. + value_counter = 0 + for value in csv_data.values: + output.write('{') + entry_counter = 0 + for entry in value: + if struct_fields[entry_counter].item_type != ITYPE_STRING: + output.write(f'{str(entry)},') + else: + output.write(f'\"{entry}\",') + entry_counter += 1 + + # Write the length of the first entry + output.write(str(original_lengths[value_counter])) + + # Close entry, add comma if not the last.. + output.write('}') + if value_counter + 1 < len(csv_data.values): + output.write(',') + + # Leave comment referring to the unhashed-value + output.write(f' // {original_names[value_counter]}') + + output.write('\n') + + value_counter += 1 + + # End struct! + output.write('};\n') + +def create_qc_structfields(csv_data): + ''' + Parses the .CSV data to create new StructField + entries given the .CSV specific requirements. + ''' + global struct_fields + + column_count = 0 + for column in csv_data.columns: + # Assume first entry is what we always want + # to hash, append _crc to it, too. + if column_count == 0: + item_type = ITYPE_CRC + item_name = column + '_crc' + else: + item_type = ITYPE_STRING + item_name = column + struct_fields.append(StructField(item_name, item_type)) + column_count += 1 + + # Always append a field that will store the + # length of the unhashed-CRC. + struct_fields.append(StructField('crc_strlen', ITYPE_FLOAT)) + + print(struct_fields) + +def generate_qc_file(csv_data): + ''' + Calls for population of StructFields and prompts + for writing the .QC file output. + ''' + create_qc_structfields(csv_data) + write_qc_file(csv_data) + +def read_csv_data(): + ''' + Parses the input_file .CSV into a Pandas dictionary, + performs the hashing on the first indexes, and sorts + in ascending order. + ''' + global original_lengths, original_names + csv_data = pandas.read_csv(args['input_file']) + + # Grab every value and turn the first entry into a hash. + for value in csv_data.values: + original_lengths.append(len(value[0])) + original_names.append(value[0]) + value[0] = int(crc16.xmodem(str.encode(value[0]))) + + # Now order everything by ascending order + csv_data = csv_data.sort_values(csv_data.columns[0]) + original_lengths = [original_lengths[i] for i in csv_data.index] + original_names = [original_names[i] for i in csv_data.index] + + return csv_data + +def fetch_cli_arguments(): + ''' + Initiates ArgParser with all potential command line arguments. + ''' + global args + parser = argparse.ArgumentParser(description='XMODEM CRC16 hash generator in FTE QuakeC-readable data structure.') + parser.add_argument('-i', '--input-file', + help='.CSV input file to parse.', required=True) + parser.add_argument('-o', '--output-file', + help='File name for generated .QC file.', default='hashes.qc') + parser.add_argument('-n', '--struct-name', + help='Name of the struct generated.', default='asset_conversion_table') + args = vars(parser.parse_args()) + +def main(): + fetch_cli_arguments() + + if not os.path.isfile(args['input_file']): + print(f'{COL_RED}Error{COL_NONE}: Input .CSV file does not exist. Exiting.') + sys.exit() + + csv_data = read_csv_data() + generate_qc_file(csv_data) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c51092c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +colorama==0.4.6 +fastcrc==0.3.0 +pandas==2.1.4