From 938badc3b0a8b71647e2463241f2e3fe8578f32a Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Mon, 17 Jun 2024 21:51:53 +0100 Subject: Generate keymap dd keycodes to header (#20273) --- lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/generate/keymap_h.py | 51 +++++++++++++++++++++++++++++++++ lib/python/qmk/keymap.py | 34 ++-------------------- 3 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 lib/python/qmk/cli/generate/keymap_h.py (limited to 'lib/python') diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 6d05a5fc21..b656909f85 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -58,6 +58,7 @@ subcommands = [ 'qmk.cli.generate.keyboard_h', 'qmk.cli.generate.keycodes', 'qmk.cli.generate.keycodes_tests', + 'qmk.cli.generate.keymap_h', 'qmk.cli.generate.make_dependencies', 'qmk.cli.generate.rgb_breathe_table', 'qmk.cli.generate.rules_mk', diff --git a/lib/python/qmk/cli/generate/keymap_h.py b/lib/python/qmk/cli/generate/keymap_h.py new file mode 100644 index 0000000000..a3aaa405c0 --- /dev/null +++ b/lib/python/qmk/cli/generate/keymap_h.py @@ -0,0 +1,51 @@ +from argcomplete.completers import FilesCompleter + +from milc import cli + +import qmk.path +from qmk.commands import dump_lines +from qmk.commands import parse_configurator_json +from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE + + +def _generate_keycodes_function(keymap_json): + """Generates keymap level keycodes. + """ + lines = [] + lines.append('enum keymap_keycodes {') + + for index, item in enumerate(keymap_json.get('keycodes', [])): + key = item["key"] + if index == 0: + lines.append(f' {key} = QK_USER_0,') + else: + lines.append(f' {key},') + + lines.append('};') + + for item in keymap_json.get('keycodes', []): + key = item["key"] + for alias in item.get("aliases", []): + lines.append(f'#define {alias} {key}') + + return lines + + +@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('filename', type=qmk.path.FileType('r'), arg_only=True, completer=FilesCompleter('.json'), help='Configurator JSON file') +@cli.subcommand('Creates a keymap.h from a QMK Configurator export.') +def generate_keymap_h(cli): + """Creates a keymap.h from a QMK Configurator export + """ + if cli.args.output and cli.args.output.name == '-': + cli.args.output = None + + keymap_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '// clang-format off'] + + keymap_json = parse_configurator_json(cli.args.filename) + + if 'keycodes' in keymap_json and keymap_json['keycodes'] is not None: + keymap_h_lines += _generate_keycodes_function(keymap_json) + + dump_lines(cli.args.output, keymap_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index b7bf897377..ca76f1b288 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -19,6 +19,9 @@ from qmk.info import info_json # The `keymap.c` template to use when a keyboard doesn't have its own DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H +#if __has_include("keymap.h") +# include "keymap.h" +#endif __INCLUDES__ /* THIS FILE WAS GENERATED! @@ -26,8 +29,6 @@ __INCLUDES__ * This file was generated by qmk json2c. You may or may not want to * edit it directly. */ -__KEYCODE_OUTPUT_GOES_HERE__ - const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { __KEYMAP_GOES_HERE__ }; @@ -125,29 +126,6 @@ def _generate_macros_function(keymap_json): return macro_txt -def _generate_keycodes_function(keymap_json): - """Generates keymap level keycodes. - """ - lines = [] - lines.append('enum keymap_keycodes {') - - for index, item in enumerate(keymap_json.get('keycodes', [])): - key = item["key"] - if index == 0: - lines.append(f' {key} = QK_USER_0,') - else: - lines.append(f' {key},') - - lines.append('};') - - for item in keymap_json.get('keycodes', []): - key = item["key"] - for alias in item.get("aliases", []): - lines.append(f'#define {alias} {key}') - - return lines - - def template_json(keyboard): """Returns a `keymap.json` template for a keyboard. @@ -350,12 +328,6 @@ def generate_c(keymap_json): hostlang = f'#include "keymap_{keymap_json["host_language"]}.h"\n#include "sendstring_{keymap_json["host_language"]}.h"\n' new_keymap = new_keymap.replace('__INCLUDES__', hostlang) - keycodes = '' - if 'keycodes' in keymap_json and keymap_json['keycodes'] is not None: - keycodes_txt = _generate_keycodes_function(keymap_json) - keycodes = '\n'.join(keycodes_txt) - new_keymap = new_keymap.replace('__KEYCODE_OUTPUT_GOES_HERE__', keycodes) - return new_keymap -- cgit v1.2.3 From 53a0cdc4467fb688faa2708fe75daa26686e918d Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Tue, 18 Jun 2024 03:44:22 +0100 Subject: Implement data driven joysticks (#22947) --- lib/python/qmk/cli/generate/keyboard_c.py | 38 ++++++++++++++++++++++++++++++- lib/python/qmk/constants.py | 2 ++ lib/python/qmk/info.py | 15 ++++++++++-- 3 files changed, 52 insertions(+), 3 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/generate/keyboard_c.py b/lib/python/qmk/cli/generate/keyboard_c.py index 5a6c967486..228b320942 100755 --- a/lib/python/qmk/cli/generate/keyboard_c.py +++ b/lib/python/qmk/cli/generate/keyboard_c.py @@ -6,7 +6,7 @@ from qmk.info import info_json from qmk.commands import dump_lines from qmk.keyboard import keyboard_completer, keyboard_folder from qmk.path import normpath -from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE +from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, JOYSTICK_AXES def _gen_led_configs(info_data): @@ -91,6 +91,41 @@ def _gen_matrix_mask(info_data): return lines +def _gen_joystick_axes(info_data): + """Convert info.json content to joystick_axes + """ + if 'axes' not in info_data.get('joystick', {}): + return [] + + axes = info_data['joystick']['axes'] + axes_keys = list(axes.keys()) + + lines = [] + lines.append('#ifdef JOYSTICK_ENABLE') + lines.append('joystick_config_t joystick_axes[JOYSTICK_AXIS_COUNT] = {') + + # loop over all available axes - injecting virtual axis for those not specified + for index, cur in enumerate(JOYSTICK_AXES): + # bail out if we have generated all requested axis + if len(axes_keys) == 0: + break + + axis = 'virtual' + if cur in axes: + axis = axes[cur] + axes_keys.remove(cur) + + if axis == 'virtual': + lines.append(f" [{index}] = JOYSTICK_AXIS_VIRTUAL,") + else: + lines.append(f" [{index}] = JOYSTICK_AXIS_IN({axis['input_pin']}, {axis['low']}, {axis['rest']}, {axis['high']}),") + + lines.append('};') + lines.append('#endif') + + return lines + + @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") @cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.c for.') @@ -105,6 +140,7 @@ def generate_keyboard_c(cli): keyboard_h_lines.extend(_gen_led_configs(kb_info_json)) keyboard_h_lines.extend(_gen_matrix_mask(kb_info_json)) + keyboard_h_lines.extend(_gen_joystick_axes(kb_info_json)) # Show the results dump_lines(cli.args.output, keyboard_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 90e4452f2b..b6f46180b2 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -320,3 +320,5 @@ LICENSE_TEXTS = [ you may not use this file except in compliance with the License. """]), ] + +JOYSTICK_AXES = ['x', 'y', 'z', 'rx', 'ry', 'rz'] diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 833271c09c..091a11854c 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -7,7 +7,7 @@ from dotty_dict import dotty from milc import cli -from qmk.constants import COL_LETTERS, ROW_LETTERS, CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS +from qmk.constants import COL_LETTERS, ROW_LETTERS, CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS, JOYSTICK_AXES from qmk.c_parse import find_layouts, parse_config_h_file, find_led_config from qmk.json_schema import deep_update, json_load, validate from qmk.keyboard import config_h, rules_mk @@ -249,8 +249,9 @@ def info_json(keyboard): info_data = _extract_rules_mk(info_data, rules_mk(str(keyboard))) info_data = _extract_config_h(info_data, config_h(str(keyboard))) - # Ensure that we have matrix row and column counts + # Ensure that we have various calculated values info_data = _matrix_size(info_data) + info_data = _joystick_axis_count(info_data) # Merge in data from info_data = _extract_led_config(info_data, str(keyboard)) @@ -800,6 +801,16 @@ def _matrix_size(info_data): return info_data +def _joystick_axis_count(info_data): + """Add info_data['joystick.axis_count'] if required + """ + if 'axes' in info_data.get('joystick', {}): + axes_keys = info_data['joystick']['axes'].keys() + info_data['joystick']['axis_count'] = max(JOYSTICK_AXES.index(a) for a in axes_keys) + 1 if axes_keys else 0 + + return info_data + + def _check_matrix(info_data): """Check the matrix to ensure that row/column count is consistent. """ -- cgit v1.2.3 From 0a5b8928202078a7a64b7fadf11b0fc25a26b4a6 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 20 Jun 2024 04:43:23 +1000 Subject: [CLI] Force `dump_lines()` to always use Unix line endings (#23954) --- lib/python/qmk/commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/python') diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 3db8353bfd..873380c28b 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -102,7 +102,9 @@ def dump_lines(output_file, lines, quiet=True): output_file.parent.mkdir(parents=True, exist_ok=True) if output_file.exists(): output_file.replace(output_file.parent / (output_file.name + '.bak')) - output_file.write_text(generated, encoding='utf-8') + with open(output_file, 'w', encoding='utf-8', newline='\n') as f: + f.write(generated) + # output_file.write_text(generated, encoding='utf-8', newline='\n') # `newline` needs Python 3.10 if not quiet: cli.log.info(f'Wrote {output_file.name} to {output_file}.') -- cgit v1.2.3 From cebe521b11347453156765055b54d81de188f1aa Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Wed, 26 Jun 2024 02:34:24 +0100 Subject: Fix docker_cmd.sh when userspace is not configured (#23997) --- lib/python/qmk/cli/userspace/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/userspace/path.py b/lib/python/qmk/cli/userspace/path.py index df4648e8c7..d0c1b544fb 100755 --- a/lib/python/qmk/cli/userspace/path.py +++ b/lib/python/qmk/cli/userspace/path.py @@ -4,5 +4,5 @@ from qmk.constants import QMK_USERSPACE @cli.subcommand('Detected path to QMK Userspace.', hidden=True) def userspace_path(cli): - print(QMK_USERSPACE) + print(QMK_USERSPACE or '') return -- cgit v1.2.3 From 3ffe8d917a7c43e56b11ada82ac57b86003719a3 Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Sun, 30 Jun 2024 03:39:49 +0100 Subject: Fix 'qmk new-keyboard' processing of development_board (#23996) --- lib/python/qmk/cli/new/keyboard.py | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py index 56bd05e1e3..b84b130f8e 100644 --- a/lib/python/qmk/cli/new/keyboard.py +++ b/lib/python/qmk/cli/new/keyboard.py @@ -14,7 +14,7 @@ from qmk.git import git_get_username from qmk.json_schema import load_jsonschema from qmk.path import keyboard from qmk.json_encoders import InfoJSONEncoder -from qmk.json_schema import deep_update, json_load +from qmk.json_schema import deep_update from qmk.constants import MCU2BOOTLOADER, QMK_FIRMWARE COMMUNITY = Path('layouts/default/') @@ -78,7 +78,7 @@ def replace_string(src, token, value): src.write_text(src.read_text().replace(token, value)) -def augment_community_info(src, dest): +def augment_community_info(config, src, dest): """Splice in any additional data into info.json """ info = json.loads(src.read_text()) @@ -86,6 +86,7 @@ def augment_community_info(src, dest): # merge community with template deep_update(info, template) + deep_update(info, config) # avoid assumptions on macro name by using the first available first_layout = next(iter(info["layouts"].values()))["layout"] @@ -105,7 +106,7 @@ def augment_community_info(src, dest): for item in first_layout: item["matrix"] = [int(item["y"]), int(item["x"])] - # finally write out the updated info.json + # finally write out the updated json dest.write_text(json.dumps(info, cls=InfoJSONEncoder, sort_keys=True)) @@ -212,15 +213,12 @@ def new_keyboard(cli): default_layout = cli.args.layout if cli.args.layout else prompt_layout() mcu = cli.args.type if cli.args.type else prompt_mcu() - # Preprocess any development_board presets + config = {} if mcu in dev_boards: - defaults_map = json_load(Path('data/mappings/defaults.hjson')) - board = defaults_map['development_board'][mcu] - - mcu = board['processor'] - bootloader = board['bootloader'] + config['development_board'] = mcu else: - bootloader = select_default_bootloader(mcu) + config['processor'] = mcu + config['bootloader'] = select_default_bootloader(mcu) detach_layout = False if default_layout == 'none of the above': @@ -231,17 +229,9 @@ def new_keyboard(cli): 'YEAR': str(date.today().year), 'KEYBOARD': kb_name, 'USER_NAME': user_name, - 'REAL_NAME': real_name, - 'LAYOUT': default_layout, - 'MCU': mcu, - 'BOOTLOADER': bootloader + 'REAL_NAME': real_name } - if cli.config.general.verbose: - cli.log.info("Creating keyboard with:") - for key, value in tokens.items(): - cli.echo(f" {key.ljust(10)}: {value}") - # begin with making the deepest folder in the tree keymaps_path = keyboard(kb_name) / 'keymaps/' keymaps_path.mkdir(parents=True) @@ -256,7 +246,7 @@ def new_keyboard(cli): # merge in infos community_info = Path(COMMUNITY / f'{default_layout}/info.json') - augment_community_info(community_info, keyboard(kb_name) / 'keyboard.json') + augment_community_info(config, community_info, keyboard(kb_name) / 'keyboard.json') # detach community layout and rename to just "LAYOUT" if detach_layout: @@ -265,5 +255,5 @@ def new_keyboard(cli): cli.log.info(f'{{fg_green}}Created a new keyboard called {{fg_cyan}}{kb_name}{{fg_green}}.{{fg_reset}}') cli.log.info(f"Build Command: {{fg_yellow}}qmk compile -kb {kb_name} -km default{{fg_reset}}.") - cli.log.info(f'Project Location: {{fg_cyan}}{QMK_FIRMWARE}/{keyboard(kb_name)}{{fg_reset}},') - cli.log.info("{{fg_yellow}}Now update the config files to match the hardware!{{fg_reset}}") + cli.log.info(f'Project Location: {{fg_cyan}}{QMK_FIRMWARE}/{keyboard(kb_name)}{{fg_reset}}.') + cli.log.info("{fg_yellow}Now update the config files to match the hardware!{fg_reset}") -- cgit v1.2.3 From bc0c69570b8a8b1d9a754a280053e49a825b24d7 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 3 Jul 2024 17:18:27 +1000 Subject: Rename encoder pins defines (#24003) --- lib/python/qmk/cli/generate/config_h.py | 4 ++-- lib/python/qmk/info.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py index fc681300a3..d613f7b92c 100755 --- a/lib/python/qmk/cli/generate/config_h.py +++ b/lib/python/qmk/cli/generate/config_h.py @@ -135,8 +135,8 @@ def generate_encoder_config(encoder_json, config_h_lines, postfix=''): b_pads.append(encoder["pin_b"]) resolutions.append(encoder.get("resolution", None)) - config_h_lines.append(generate_define(f'ENCODERS_PAD_A{postfix}', f'{{ {", ".join(a_pads)} }}')) - config_h_lines.append(generate_define(f'ENCODERS_PAD_B{postfix}', f'{{ {", ".join(b_pads)} }}')) + config_h_lines.append(generate_define(f'ENCODER_A_PINS{postfix}', f'{{ {", ".join(a_pads)} }}')) + config_h_lines.append(generate_define(f'ENCODER_B_PINS{postfix}', f'{{ {", ".join(b_pads)} }}')) if None in resolutions: cli.log.debug(f"Unable to generate ENCODER_RESOLUTION{postfix} configuration") diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 091a11854c..5b3b249015 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -375,8 +375,8 @@ def _extract_audio(info_data, config_c): def _extract_encoders_values(config_c, postfix=''): """Common encoder extraction logic """ - a_pad = config_c.get(f'ENCODERS_PAD_A{postfix}', '').replace(' ', '')[1:-1] - b_pad = config_c.get(f'ENCODERS_PAD_B{postfix}', '').replace(' ', '')[1:-1] + a_pad = config_c.get(f'ENCODER_A_PINS{postfix}', '').replace(' ', '')[1:-1] + b_pad = config_c.get(f'ENCODER_B_PINS{postfix}', '').replace(' ', '')[1:-1] resolutions = config_c.get(f'ENCODER_RESOLUTIONS{postfix}', '').replace(' ', '')[1:-1] default_resolution = config_c.get('ENCODER_RESOLUTION', None) -- cgit v1.2.3 From bc8ac8642242516e0fe61f1b97b75374c9150647 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Wed, 3 Jul 2024 22:00:53 +1000 Subject: Minimum python version listing. (#23989) --- lib/python/qmk/cli/__init__.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index b504aa5f8c..f331445870 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -189,20 +189,22 @@ def _eprint(errmsg): # Supported version information # # Based on the OSes we support these are the minimum python version available by default. -# Last update: 2021 Jan 02 +# Last update: 2024 Jun 24 # -# Arch: 3.9 -# Debian: 3.7 -# Fedora 31: 3.7 -# Fedora 32: 3.8 -# Fedora 33: 3.9 -# FreeBSD: 3.7 -# Gentoo: 3.7 -# macOS: 3.9 (from homebrew) -# msys2: 3.8 -# Slackware: 3.7 -# solus: 3.7 -# void: 3.9 +# Arch: 3.12 +# Debian 11: 3.9 +# Debian 12: 3.11 +# Fedora 39: 3.12 +# Fedora 40: 3.12 +# FreeBSD: 3.11 +# Gentoo: 3.12 +# macOS: 3.12 (from homebrew) +# msys2: 3.11 +# Slackware: 3.9 +# solus: 3.10 +# Ubuntu 22.04: 3.10 +# Ubuntu 24.04: 3.12 +# void: 3.12 if sys.version_info[0] != 3 or sys.version_info[1] < 7: _eprint('Error: Your Python is too old! Please upgrade to Python 3.7 or later.') -- cgit v1.2.3 From 63ef6516d31149a01df1694907425a99e06e8bcc Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Wed, 17 Jul 2024 20:31:18 +0100 Subject: Avoid path issues with `qmk flash` on Windows (#24130) --- lib/python/qmk/flashers.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/python') diff --git a/lib/python/qmk/flashers.py b/lib/python/qmk/flashers.py index 9ecb5e4b9c..7ee1bc8de7 100644 --- a/lib/python/qmk/flashers.py +++ b/lib/python/qmk/flashers.py @@ -206,6 +206,8 @@ def _flash_uf2(file): def flasher(mcu, file): + # Avoid "expected string or bytes-like object, got 'WindowsPath" issues + file = file.as_posix() bl, details = _find_bootloader() # Add a small sleep to avoid race conditions time.sleep(1) -- cgit v1.2.3 From 4ab36df48fba751ca88beaf1a7a5dd84ba34a370 Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Thu, 18 Jul 2024 00:02:53 +0100 Subject: Move split.soft_serial_pin to split.serial.pin (#24127) --- lib/python/qmk/info.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib/python') diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 5b3b249015..5948b66b5e 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -461,6 +461,14 @@ def _extract_split_handedness(info_data, config_c): split['handedness']['matrix_grid'] = split.pop('matrix_grid') +def _extract_split_serial(info_data, config_c): + # Migrate + split = info_data.get('split', {}) + if 'soft_serial_pin' in split: + split['serial'] = split.get('serial', {}) + split['serial']['pin'] = split.pop('soft_serial_pin') + + def _extract_split_transport(info_data, config_c): # Figure out the transport method if config_c.get('USE_I2C') is True: @@ -656,6 +664,7 @@ def _extract_config_h(info_data, config_c): _extract_audio(info_data, config_c) _extract_secure_unlock(info_data, config_c) _extract_split_handedness(info_data, config_c) + _extract_split_serial(info_data, config_c) _extract_split_transport(info_data, config_c) _extract_split_right_pins(info_data, config_c) _extract_encoders(info_data, config_c) -- cgit v1.2.3 From 8c35011d0aa242e9f6271d3a66ae6924d52bbed6 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Wed, 24 Jul 2024 15:16:51 +1000 Subject: [CLI] Only generate files if contents change. (#24038) Don't overwrite if the content doesn't change. --- lib/python/qmk/commands.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib/python') diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 3db8353bfd..df6ed6b88a 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -101,6 +101,12 @@ def dump_lines(output_file, lines, quiet=True): if output_file and output_file.name != '-': output_file.parent.mkdir(parents=True, exist_ok=True) if output_file.exists(): + with open(output_file, 'r', encoding='utf-8', newline='\n') as f: + existing = f.read() + if existing == generated: + if not quiet: + cli.log.info(f'No changes to {output_file.name}.') + return output_file.replace(output_file.parent / (output_file.name + '.bak')) output_file.write_text(generated, encoding='utf-8') -- cgit v1.2.3 From d538451adb2a49bc1401aa90f4a66814dd723642 Mon Sep 17 00:00:00 2001 From: Dasky <32983009+daskygit@users.noreply.github.com> Date: Sun, 28 Jul 2024 19:16:15 +0100 Subject: Remove AVR GCC version warning (#24206) --- lib/python/qmk/cli/doctor/check.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py index cd69cdd11c..d563811aba 100644 --- a/lib/python/qmk/cli/doctor/check.py +++ b/lib/python/qmk/cli/doctor/check.py @@ -53,11 +53,6 @@ def _check_avr_gcc_version(): version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip() cli.log.info('Found avr-gcc version %s', version_number) - parsed_version = _parse_gcc_version(version_number) - if parsed_version['major'] > 8: - cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.') - return CheckStatus.WARNING - return CheckStatus.OK -- cgit v1.2.3 From 9f387f525cb7f9099c4a90ad42d1195c1901c2a3 Mon Sep 17 00:00:00 2001 From: Ryan Date: Sun, 4 Aug 2024 01:55:02 +1000 Subject: Remove `handwired/pytest/has_template` (#24232) --- lib/python/qmk/tests/test_cli_commands.py | 78 +++++++++++++++++++++++-------- lib/python/qmk/tests/test_qmk_keymap.py | 45 +++++++++++------- 2 files changed, 87 insertions(+), 36 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index 8b50d1c340..674bb54be2 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -142,9 +142,34 @@ def test_list_keymaps_no_keyboard_found(): def test_json2c(): - result = check_subcommand('json2c', 'keyboards/handwired/pytest/has_template/keymaps/default_json/keymap.json') + result = check_subcommand('json2c', 'keyboards/handwired/pytest/basic/keymaps/default_json/keymap.json') check_returncode(result) - assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n\n' + assert result.stdout == """#include QMK_KEYBOARD_H +#if __has_include("keymap.h") +# include "keymap.h" +#endif + + +/* THIS FILE WAS GENERATED! + * + * This file was generated by qmk json2c. You may or may not want to + * edit it directly. + */ +const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { +\t[0] = LAYOUT_ortho_1x1(KC_A) +}; + +#if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) +const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][NUM_DIRECTIONS] = { + +}; +#endif // defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) + + + + + +""" def test_json2c_macros(): @@ -156,9 +181,34 @@ def test_json2c_macros(): def test_json2c_stdin(): - result = check_subcommand_stdin('keyboards/handwired/pytest/has_template/keymaps/default_json/keymap.json', 'json2c', '-') + result = check_subcommand_stdin('keyboards/handwired/pytest/basic/keymaps/default_json/keymap.json', 'json2c', '-') check_returncode(result) - assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n\n' + assert result.stdout == """#include QMK_KEYBOARD_H +#if __has_include("keymap.h") +# include "keymap.h" +#endif + + +/* THIS FILE WAS GENERATED! + * + * This file was generated by qmk json2c. You may or may not want to + * edit it directly. + */ +const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { +\t[0] = LAYOUT_ortho_1x1(KC_A) +}; + +#if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) +const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][NUM_DIRECTIONS] = { + +}; +#endif // defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) + + + + + +""" def test_json2c_wrong_json(): @@ -223,27 +273,15 @@ def test_info_matrix_render(): def test_c2json(): - result = check_subcommand("c2json", "-kb", "handwired/pytest/has_template", "-km", "default", "keyboards/handwired/pytest/has_template/keymaps/default/keymap.c") + result = check_subcommand("c2json", "-kb", "handwired/pytest/basic", "-km", "default", "keyboards/handwired/pytest/basic/keymaps/default/keymap.c") check_returncode(result) - assert result.stdout.strip() == '{"keyboard": "handwired/pytest/has_template", "documentation": "This file is a keymap.json file for handwired/pytest/has_template", "keymap": "default", "layout": "LAYOUT_ortho_1x1", "layers": [["KC_A"]]}' - - -def test_c2json_nocpp(): - result = check_subcommand("c2json", "--no-cpp", "-kb", "handwired/pytest/has_template", "-km", "default", "keyboards/handwired/pytest/has_template/keymaps/nocpp/keymap.c") - check_returncode(result) - assert result.stdout.strip() == '{"keyboard": "handwired/pytest/has_template", "documentation": "This file is a keymap.json file for handwired/pytest/has_template", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_ENTER"]]}' + assert result.stdout.strip() == '{"keyboard": "handwired/pytest/basic", "keymap": "default", "layout": "LAYOUT_ortho_1x1", "layers": [["KC_A"]]}' def test_c2json_stdin(): - result = check_subcommand_stdin("keyboards/handwired/pytest/has_template/keymaps/default/keymap.c", "c2json", "-kb", "handwired/pytest/has_template", "-km", "default", "-") - check_returncode(result) - assert result.stdout.strip() == '{"keyboard": "handwired/pytest/has_template", "documentation": "This file is a keymap.json file for handwired/pytest/has_template", "keymap": "default", "layout": "LAYOUT_ortho_1x1", "layers": [["KC_A"]]}' - - -def test_c2json_nocpp_stdin(): - result = check_subcommand_stdin("keyboards/handwired/pytest/has_template/keymaps/nocpp/keymap.c", "c2json", "--no-cpp", "-kb", "handwired/pytest/has_template", "-km", "default", "-") + result = check_subcommand_stdin("keyboards/handwired/pytest/basic/keymaps/default/keymap.c", "c2json", "-kb", "handwired/pytest/basic", "-km", "default", "-") check_returncode(result) - assert result.stdout.strip() == '{"keyboard": "handwired/pytest/has_template", "documentation": "This file is a keymap.json file for handwired/pytest/has_template", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_ENTER"]]}' + assert result.stdout.strip() == '{"keyboard": "handwired/pytest/basic", "keymap": "default", "layout": "LAYOUT_ortho_1x1", "layers": [["KC_A"]]}' def test_clean(): diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py index 5e2efc1232..b4b8f4bbe0 100644 --- a/lib/python/qmk/tests/test_qmk_keymap.py +++ b/lib/python/qmk/tests/test_qmk_keymap.py @@ -11,30 +11,43 @@ def test_template_json_pytest_basic(): assert templ == {'keyboard': 'handwired/pytest/basic'} -def test_template_c_pytest_has_template(): - templ = qmk.keymap.template_c('handwired/pytest/has_template') - assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {__KEYMAP_GOES_HERE__};\n' - - -def test_template_json_pytest_has_template(): - templ = qmk.keymap.template_json('handwired/pytest/has_template') - assert templ == {'keyboard': 'handwired/pytest/has_template', "documentation": "This file is a keymap.json file for handwired/pytest/has_template"} - - -def test_generate_c_pytest_has_template(): +def test_generate_c_pytest_basic(): keymap_json = { - 'keyboard': 'handwired/pytest/has_template', + 'keyboard': 'handwired/pytest/basic', 'layout': 'LAYOUT', 'layers': [['KC_A']], 'macros': None, } templ = qmk.keymap.generate_c(keymap_json) - assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n' + assert templ == """#include QMK_KEYBOARD_H +#if __has_include("keymap.h") +# include "keymap.h" +#endif + + +/* THIS FILE WAS GENERATED! + * + * This file was generated by qmk json2c. You may or may not want to + * edit it directly. + */ +const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { +\t[0] = LAYOUT(KC_A) +}; + +#if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) +const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][NUM_DIRECTIONS] = { + +}; +#endif // defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) + + + +""" -def test_generate_json_pytest_has_template(): - templ = qmk.keymap.generate_json('default', 'handwired/pytest/has_template', 'LAYOUT', [['KC_A']]) - assert templ == {"keyboard": "handwired/pytest/has_template", "documentation": "This file is a keymap.json file for handwired/pytest/has_template", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_A"]]} +def test_generate_json_pytest_basic(): + templ = qmk.keymap.generate_json('default', 'handwired/pytest/basic', 'LAYOUT', [['KC_A']]) + assert templ == {"keyboard": "handwired/pytest/basic", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_A"]]} def test_parse_keymap_c(): -- cgit v1.2.3 From 1f942bb17e0f324f81788106a72c7573304d65ff Mon Sep 17 00:00:00 2001 From: Ryan Date: Sun, 4 Aug 2024 21:28:40 +1000 Subject: Small tweaks to keymap generation (#24240) --- lib/python/qmk/keymap.py | 5 ++--- lib/python/qmk/tests/test_cli_commands.py | 6 ++---- lib/python/qmk/tests/test_qmk_keymap.py | 3 +-- 3 files changed, 5 insertions(+), 9 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index ca76f1b288..8c09cc51dc 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -40,7 +40,6 @@ __ENCODER_MAP_GOES_HERE__ #endif // defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) __MACRO_OUTPUT_GOES_HERE__ - """ @@ -51,7 +50,7 @@ def _generate_keymap_table(keymap_json): lines[-1] = lines[-1] + ',' layer = map(_strip_any, layer) layer_keys = ', '.join(layer) - lines.append('\t[%s] = %s(%s)' % (layer_num, keymap_json['layout'], layer_keys)) + lines.append(' [%s] = %s(%s)' % (layer_num, keymap_json['layout'], layer_keys)) return lines @@ -61,7 +60,7 @@ def _generate_encodermap_table(keymap_json): if layer_num != 0: lines[-1] = lines[-1] + ',' encoder_keycode_txt = ', '.join([f'ENCODER_CCW_CW({_strip_any(e["ccw"])}, {_strip_any(e["cw"])})' for e in layer]) - lines.append('\t[%s] = {%s}' % (layer_num, encoder_keycode_txt)) + lines.append(' [%s] = {%s}' % (layer_num, encoder_keycode_txt)) return lines diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index 674bb54be2..2b3d6f4049 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -156,7 +156,7 @@ def test_json2c(): * edit it directly. */ const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { -\t[0] = LAYOUT_ortho_1x1(KC_A) + [0] = LAYOUT_ortho_1x1(KC_A) }; #if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) @@ -168,7 +168,6 @@ const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][NUM_DIRECTIONS] = { - """ @@ -195,7 +194,7 @@ def test_json2c_stdin(): * edit it directly. */ const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { -\t[0] = LAYOUT_ortho_1x1(KC_A) + [0] = LAYOUT_ortho_1x1(KC_A) }; #if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) @@ -207,7 +206,6 @@ const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][NUM_DIRECTIONS] = { - """ diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py index b4b8f4bbe0..bec3f4006c 100644 --- a/lib/python/qmk/tests/test_qmk_keymap.py +++ b/lib/python/qmk/tests/test_qmk_keymap.py @@ -31,7 +31,7 @@ def test_generate_c_pytest_basic(): * edit it directly. */ const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { -\t[0] = LAYOUT(KC_A) + [0] = LAYOUT(KC_A) }; #if defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) @@ -41,7 +41,6 @@ const uint16_t PROGMEM encoder_map[][NUM_ENCODERS][NUM_DIRECTIONS] = { #endif // defined(ENCODER_ENABLE) && defined(ENCODER_MAP_ENABLE) - """ -- cgit v1.2.3 From 783f97ff32de1d6febceb09f46dfa624e4fc56ec Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Mon, 12 Aug 2024 13:29:05 +0100 Subject: Remove handling of keyboard level keymap templates (#24234) --- lib/python/qmk/keymap.py | 39 ++------------------------------- lib/python/qmk/tests/test_qmk_keymap.py | 10 --------- 2 files changed, 2 insertions(+), 47 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 8c09cc51dc..f3505d324e 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -125,41 +125,6 @@ def _generate_macros_function(keymap_json): return macro_txt -def template_json(keyboard): - """Returns a `keymap.json` template for a keyboard. - - If a template exists in `keyboards//templates/keymap.json` that text will be used instead of an empty dictionary. - - Args: - keyboard - The keyboard to return a template for. - """ - template_file = Path('keyboards/%s/templates/keymap.json' % keyboard) - template = {'keyboard': keyboard} - if template_file.exists(): - template.update(json.load(template_file.open(encoding='utf-8'))) - - return template - - -def template_c(keyboard): - """Returns a `keymap.c` template for a keyboard. - - If a template exists in `keyboards//templates/keymap.c` that text will be used instead of an empty dictionary. - - Args: - keyboard - The keyboard to return a template for. - """ - template_file = Path('keyboards/%s/templates/keymap.c' % keyboard) - if template_file.exists(): - template = template_file.read_text(encoding='utf-8') - else: - template = DEFAULT_KEYMAP_C - - return template - - def _strip_any(keycode): """Remove ANY() from a keycode. """ @@ -278,7 +243,7 @@ def generate_json(keymap, keyboard, layout, layers, macros=None): macros A sequence of strings containing macros to implement for this keyboard. """ - new_keymap = template_json(keyboard) + new_keymap = {'keyboard': keyboard} new_keymap['keymap'] = keymap new_keymap['layout'] = layout new_keymap['layers'] = layers @@ -305,7 +270,7 @@ def generate_c(keymap_json): macros A sequence of strings containing macros to implement for this keyboard. """ - new_keymap = template_c(keymap_json['keyboard']) + new_keymap = DEFAULT_KEYMAP_C layer_txt = _generate_keymap_table(keymap_json) keymap = '\n'.join(layer_txt) new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap) diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py index bec3f4006c..7482848eff 100644 --- a/lib/python/qmk/tests/test_qmk_keymap.py +++ b/lib/python/qmk/tests/test_qmk_keymap.py @@ -1,16 +1,6 @@ import qmk.keymap -def test_template_c_pytest_basic(): - templ = qmk.keymap.template_c('handwired/pytest/basic') - assert templ == qmk.keymap.DEFAULT_KEYMAP_C - - -def test_template_json_pytest_basic(): - templ = qmk.keymap.template_json('handwired/pytest/basic') - assert templ == {'keyboard': 'handwired/pytest/basic'} - - def test_generate_c_pytest_basic(): keymap_json = { 'keyboard': 'handwired/pytest/basic', -- cgit v1.2.3 From 380e0c9cad72ac29f858bef85c8b8eb35b6931f0 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Mon, 12 Aug 2024 22:34:22 +1000 Subject: Userspace: add support for adding environment variables during build (#22887) --- lib/python/qmk/build_targets.py | 107 ++++++++++++++------- lib/python/qmk/cli/format/json.py | 9 +- lib/python/qmk/cli/mass_compile.py | 24 +++-- lib/python/qmk/cli/userspace/add.py | 10 +- lib/python/qmk/cli/userspace/compile.py | 10 +- lib/python/qmk/cli/userspace/list.py | 23 ++++- lib/python/qmk/cli/userspace/remove.py | 10 +- lib/python/qmk/commands.py | 7 +- lib/python/qmk/info.py | 19 ++-- lib/python/qmk/keymap.py | 4 +- lib/python/qmk/search.py | 159 ++++++++++++++++++++------------ lib/python/qmk/userspace.py | 66 +++++++++---- 12 files changed, 304 insertions(+), 144 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py index d974d04020..e2df029490 100644 --- a/lib/python/qmk/build_targets.py +++ b/lib/python/qmk/build_targets.py @@ -1,8 +1,8 @@ -# Copyright 2023 Nick Brassel (@tzarc) +# Copyright 2023-2024 Nick Brassel (@tzarc) # SPDX-License-Identifier: GPL-2.0-or-later import json import shutil -from typing import List, Union +from typing import Dict, List, Union from pathlib import Path from dotty_dict import dotty, Dotty from milc import cli @@ -13,6 +13,9 @@ from qmk.info import keymap_json from qmk.keymap import locate_keymap from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace +# These must be kept in the order in which they're applied to $(TARGET) in the makefiles in order to ensure consistency. +TARGET_FILENAME_MODIFIERS = ['FORCE_LAYOUT', 'CONVERT_TO'] + class BuildTarget: def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None): @@ -22,25 +25,25 @@ class BuildTarget: self._parallel = 1 self._clean = False self._compiledb = False - self._target = f'{self._keyboard_safe}_{self.keymap}' - self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}') - self._generated_files_path = self._intermediate_output / 'src' + self._extra_args = {} self._json = json.to_dict() if isinstance(json, Dotty) else json def __str__(self): return f'{self.keyboard}:{self.keymap}' def __repr__(self): + if len(self._extra_args.items()) > 0: + return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={json.dumps(self._extra_args, sort_keys=True)})' return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})' + def __lt__(self, __value: object) -> bool: + return self.__repr__() < __value.__repr__() + def __eq__(self, __value: object) -> bool: if not isinstance(__value, BuildTarget): return False return self.__repr__() == __value.__repr__() - def __ne__(self, __value: object) -> bool: - return not self.__eq__(__value) - def __hash__(self) -> int: return self.__repr__().__hash__() @@ -72,7 +75,34 @@ class BuildTarget: def dotty(self) -> Dotty: return dotty(self.json) - def _common_make_args(self, dry_run: bool = False, build_target: str = None): + @property + def extra_args(self) -> Dict[str, str]: + return {k: v for k, v in self._extra_args.items()} + + @extra_args.setter + def extra_args(self, ex_args: Dict[str, str]): + if ex_args is not None and isinstance(ex_args, dict): + self._extra_args = {k: v for k, v in ex_args.items()} + + def target_name(self, **env_vars) -> str: + # Work out the intended target name + target = f'{self._keyboard_safe}_{self.keymap}' + vars = self._all_vars(**env_vars) + for modifier in TARGET_FILENAME_MODIFIERS: + if modifier in vars: + target += f"_{vars[modifier]}" + return target + + def _all_vars(self, **env_vars) -> Dict[str, str]: + vars = {k: v for k, v in env_vars.items()} + for k, v in self._extra_args.items(): + vars[k] = v + return vars + + def _intermediate_output(self, **env_vars) -> Path: + return Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self.target_name(**env_vars)}') + + def _common_make_args(self, dry_run: bool = False, build_target: str = None, **env_vars): compile_args = [ find_make(), *get_make_parallel_args(self._parallel), @@ -98,14 +128,17 @@ class BuildTarget: f'KEYBOARD={self.keyboard}', f'KEYMAP={self.keymap}', f'KEYBOARD_FILESAFE={self._keyboard_safe}', - f'TARGET={self._target}', - f'INTERMEDIATE_OUTPUT={self._intermediate_output}', + f'TARGET={self._keyboard_safe}_{self.keymap}', # don't use self.target_name() here, it's rebuilt on the makefile side f'VERBOSE={verbose}', f'COLOR={color}', 'SILENT=false', 'QMK_BIN="qmk"', ]) + vars = self._all_vars(**env_vars) + for k, v in vars.items(): + compile_args.append(f'{k}={v}') + return compile_args def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: @@ -150,6 +183,8 @@ class KeyboardKeymapBuildTarget(BuildTarget): super().__init__(keyboard=keyboard, keymap=keymap, json=json) def __repr__(self): + if len(self._extra_args.items()) > 0: + return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={self._extra_args})' return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})' def _load_json(self): @@ -159,15 +194,13 @@ class KeyboardKeymapBuildTarget(BuildTarget): pass def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: - compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target) - - for key, value in env_vars.items(): - compile_args.append(f'{key}={value}') + compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target, **env_vars) # Need to override the keymap path if the keymap is a userspace directory. # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap # in an equivalent historical location. - keymap_location = locate_keymap(self.keyboard, self.keymap) + vars = self._all_vars(**env_vars) + keymap_location = locate_keymap(self.keyboard, self.keymap, force_layout=vars.get('FORCE_LAYOUT')) if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location): keymap_directory = keymap_location.parent compile_args.extend([ @@ -196,47 +229,51 @@ class JsonKeymapBuildTarget(BuildTarget): super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json) - self._keymap_json = self._generated_files_path / 'keymap.json' - def __repr__(self): + if len(self._extra_args.items()) > 0: + return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path}, extra_args={self._extra_args})' return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})' def _load_json(self): pass # Already loaded in constructor def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: + intermediate_output = self._intermediate_output(**env_vars) + generated_files_path = intermediate_output / 'src' + keymap_json = generated_files_path / 'keymap.json' + if self._clean: - if self._intermediate_output.exists(): - shutil.rmtree(self._intermediate_output) + if intermediate_output.exists(): + shutil.rmtree(intermediate_output) # begin with making the deepest folder in the tree - self._generated_files_path.mkdir(exist_ok=True, parents=True) + generated_files_path.mkdir(exist_ok=True, parents=True) # Compare minified to ensure consistent comparison new_content = json.dumps(self.json, separators=(',', ':')) - if self._keymap_json.exists(): - old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) + if keymap_json.exists(): + old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) if old_content == new_content: new_content = None # Write the keymap.json file if different so timestamps are only updated # if the content changes -- running `make` won't treat it as modified. if new_content: - self._keymap_json.write_text(new_content, encoding='utf-8') + keymap_json.write_text(new_content, encoding='utf-8') def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: - compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target) + compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target, **env_vars) + intermediate_output = self._intermediate_output(**env_vars) + generated_files_path = intermediate_output / 'src' + keymap_json = generated_files_path / 'keymap.json' compile_args.extend([ - f'MAIN_KEYMAP_PATH_1={self._intermediate_output}', - f'MAIN_KEYMAP_PATH_2={self._intermediate_output}', - f'MAIN_KEYMAP_PATH_3={self._intermediate_output}', - f'MAIN_KEYMAP_PATH_4={self._intermediate_output}', - f'MAIN_KEYMAP_PATH_5={self._intermediate_output}', - f'KEYMAP_JSON={self._keymap_json}', - f'KEYMAP_PATH={self._generated_files_path}', + f'MAIN_KEYMAP_PATH_1={intermediate_output}', + f'MAIN_KEYMAP_PATH_2={intermediate_output}', + f'MAIN_KEYMAP_PATH_3={intermediate_output}', + f'MAIN_KEYMAP_PATH_4={intermediate_output}', + f'MAIN_KEYMAP_PATH_5={intermediate_output}', + f'KEYMAP_JSON={keymap_json}', + f'KEYMAP_PATH={generated_files_path}', ]) - for key, value in env_vars.items(): - compile_args.append(f'{key}={value}') - return compile_args diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py index 87a3837d10..3670294434 100755 --- a/lib/python/qmk/cli/format/json.py +++ b/lib/python/qmk/cli/format/json.py @@ -18,11 +18,18 @@ def _detect_json_format(file, json_data): """ json_encoder = None try: - validate(json_data, 'qmk.user_repo.v1') + validate(json_data, 'qmk.user_repo.v1_1') json_encoder = UserspaceJSONEncoder except ValidationError: pass + if json_encoder is None: + try: + validate(json_data, 'qmk.user_repo.v1') + json_encoder = UserspaceJSONEncoder + except ValidationError: + pass + if json_encoder is None: try: validate(json_data, 'qmk.keyboard.v1') diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py index d13afc6143..cf9be0fd1e 100755 --- a/lib/python/qmk/cli/mass_compile.py +++ b/lib/python/qmk/cli/mass_compile.py @@ -7,6 +7,7 @@ from typing import List from pathlib import Path from subprocess import DEVNULL from milc import cli +import shlex from qmk.constants import QMK_FIRMWARE from qmk.commands import find_make, get_make_parallel_args, build_environment @@ -26,7 +27,8 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, if dry_run: cli.log.info('Compilation targets:') for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): - cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap}{{fg_reset}}") + extra_args = ' '.join([f"-e {shlex.quote(f'{k}={v}')}" for k, v in target.extra_args.items()]) + cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap} {extra_args}{{fg_reset}}") else: if clean: cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL) @@ -36,18 +38,26 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): keyboard_name = target.keyboard keymap_name = target.keymap + keyboard_safe = keyboard_name.replace('/', '_') + target_filename = target.target_name(**env) target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make` command = target.compile_command(**env) command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism - keyboard_safe = keyboard_name.replace('/', '_') + extra_args = '_'.join([f"{k}_{v}" for k, v in target.extra_args.items()]) build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" + target_suffix = '' + if len(extra_args) > 0: + build_log += f".{extra_args}" + failed_log += f".{extra_args}" + target_suffix = f"_{extra_args}" # yapf: disable f.write( f"""\ -all: {keyboard_safe}_{keymap_name}_binary -{keyboard_safe}_{keymap_name}_binary: +.PHONY: {target_filename}{target_suffix}_binary +all: {target_filename}{target_suffix}_binary +{target_filename}{target_suffix}_binary: @rm -f "{build_log}" || true @echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}" {' '.join(command)} \\ @@ -65,9 +75,9 @@ all: {keyboard_safe}_{keymap_name}_binary # yapf: disable f.write( f"""\ - @rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.elf" 2>/dev/null || true - @rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.map" 2>/dev/null || true - @rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{keymap_name}" || true + @rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.elf" 2>/dev/null || true + @rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.map" 2>/dev/null || true + @rm -rf "{QMK_FIRMWARE}/.build/obj_{target_filename}" || true """# noqa ) # yapf: enable diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py index 8993d54dba..0d6f32cd11 100644 --- a/lib/python/qmk/cli/userspace/add.py +++ b/lib/python/qmk/cli/userspace/add.py @@ -1,8 +1,9 @@ -# Copyright 2023 Nick Brassel (@tzarc) +# Copyright 2023-2024 Nick Brassel (@tzarc) # SPDX-License-Identifier: GPL-2.0-or-later from pathlib import Path from milc import cli +from qmk.commands import parse_env_vars from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE from qmk.keyboard import keyboard_completer, keyboard_folder_or_all from qmk.keymap import keymap_completer, is_keymap_target @@ -12,12 +13,15 @@ from qmk.userspace import UserspaceDefs @cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form :, or path to a keymap JSON file.") @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Extra variables to set during build. May be passed multiple times.") @cli.subcommand('Adds a build target to userspace `qmk.json`.') def userspace_add(cli): if not HAS_QMK_USERSPACE: cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') return False + build_env = None if len(cli.args.env) == 0 else parse_env_vars(cli.args.env) + userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') if len(cli.args.builds) > 0: @@ -44,8 +48,8 @@ def userspace_add(cli): cli.config.new_keymap.keyboard = cli.args.keyboard cli.config.new_keymap.keymap = cli.args.keymap if new_keymap(cli) is not False: - userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env) else: - userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env) return userspace.save() diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py index e8cdf6cd97..f164ca2ef1 100644 --- a/lib/python/qmk/cli/userspace/compile.py +++ b/lib/python/qmk/cli/userspace/compile.py @@ -1,4 +1,4 @@ -# Copyright 2023 Nick Brassel (@tzarc) +# Copyright 2023-2024 Nick Brassel (@tzarc) # SPDX-License-Identifier: GPL-2.0-or-later from pathlib import Path from milc import cli @@ -12,6 +12,10 @@ from qmk.cli.mass_compile import mass_compile_targets from qmk.util import maybe_exit_config +def _extra_arg_setter(target, extra_args): + target.extra_args = extra_args + + @cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.") @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @@ -33,8 +37,8 @@ def userspace_compile(cli): if isinstance(e, Path): build_targets.append(JsonKeymapBuildTarget(e)) elif isinstance(e, dict): - keyboard_keymap_targets.append((e['keyboard'], e['keymap'])) - + f = e['env'] if 'env' in e else None + keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f)) if len(keyboard_keymap_targets) > 0: build_targets.extend(search_keymap_targets(keyboard_keymap_targets)) diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py index 8689c80a76..9f83a14a2a 100644 --- a/lib/python/qmk/cli/userspace/list.py +++ b/lib/python/qmk/cli/userspace/list.py @@ -1,4 +1,4 @@ -# Copyright 2023 Nick Brassel (@tzarc) +# Copyright 2023-2024 Nick Brassel (@tzarc) # SPDX-License-Identifier: GPL-2.0-or-later from pathlib import Path from dotty_dict import Dotty @@ -13,6 +13,10 @@ from qmk.search import search_keymap_targets from qmk.util import maybe_exit_config +def _extra_arg_setter(target, extra_args): + target.extra_args = extra_args + + @cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.") @cli.subcommand('Lists the build targets specified in userspace `qmk.json`.') def userspace_list(cli): @@ -26,11 +30,15 @@ def userspace_list(cli): if cli.args.expand: build_targets = [] + keyboard_keymap_targets = [] for e in userspace.build_targets: if isinstance(e, Path): build_targets.append(e) elif isinstance(e, dict) or isinstance(e, Dotty): - build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])])) + f = e['env'] if 'env' in e else None + keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f)) + if len(keyboard_keymap_targets) > 0: + build_targets.extend(search_keymap_targets(keyboard_keymap_targets)) else: build_targets = userspace.build_targets @@ -43,12 +51,19 @@ def userspace_list(cli): # keyboard/keymap dict from userspace keyboard = e['keyboard'] keymap = e['keymap'] + extra_args = e.get('env') elif isinstance(e, BuildTarget): # BuildTarget from search_keymap_targets() keyboard = e.keyboard keymap = e.keymap + extra_args = e.extra_args + + extra_args_str = '' + if extra_args is not None and len(extra_args) > 0: + extra_args_str = ', '.join([f'{{fg_cyan}}{k}={v}{{fg_reset}}' for k, v in extra_args.items()]) + extra_args_str = f' ({{fg_cyan}}{extra_args_str}{{fg_reset}})' if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap): - cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}') + cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}{extra_args_str}') else: - cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!') + cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}{extra_args_str} -- not found!') diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py index c7d180bfd1..b2da08a98e 100644 --- a/lib/python/qmk/cli/userspace/remove.py +++ b/lib/python/qmk/cli/userspace/remove.py @@ -1,8 +1,9 @@ -# Copyright 2023 Nick Brassel (@tzarc) +# Copyright 2023-2024 Nick Brassel (@tzarc) # SPDX-License-Identifier: GPL-2.0-or-later from pathlib import Path from milc import cli +from qmk.commands import parse_env_vars from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE from qmk.keyboard import keyboard_completer, keyboard_folder_or_all from qmk.keymap import keymap_completer @@ -12,12 +13,15 @@ from qmk.userspace import UserspaceDefs @cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form :, or path to a keymap JSON file.") @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Extra variables to set during build. May be passed multiple times.") @cli.subcommand('Removes a build target from userspace `qmk.json`.') def userspace_remove(cli): if not HAS_QMK_USERSPACE: cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') return False + build_env = None if len(cli.args.env) == 0 else parse_env_vars(cli.args.env) + userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') if len(cli.args.builds) > 0: @@ -29,9 +33,9 @@ def userspace_remove(cli): for e in make_like_targets: s = e.split(':') - userspace.remove_target(keyboard=s[0], keymap=s[1]) + userspace.remove_target(keyboard=s[0], keymap=s[1], build_env=build_env) else: - userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env) return userspace.save() diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index df6ed6b88a..459858ab2c 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -68,7 +68,7 @@ def parse_configurator_json(configurator_file): return user_keymap -def build_environment(args): +def parse_env_vars(args): """Common processing for cli.args.env """ envs = {} @@ -78,6 +78,11 @@ def build_environment(args): envs[key] = value else: cli.log.warning('Invalid environment variable: %s', env) + return envs + + +def build_environment(args): + envs = parse_env_vars(args) if HAS_QMK_USERSPACE: envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve() diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 833271c09c..e295c5cfe0 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -212,7 +212,7 @@ def _validate(keyboard, info_data): maybe_exit(1) -def info_json(keyboard): +def info_json(keyboard, force_layout=None): """Generate the info.json data for a specific keyboard. """ cur_dir = Path('keyboards') @@ -255,6 +255,11 @@ def info_json(keyboard): # Merge in data from info_data = _extract_led_config(info_data, str(keyboard)) + # Force a community layout if requested + community_layouts = info_data.get("community_layouts", []) + if force_layout in community_layouts: + info_data["community_layouts"] = [force_layout] + # Validate _validate(keyboard, info_data) @@ -988,25 +993,25 @@ def find_info_json(keyboard): return [info_json for info_json in info_jsons if info_json.exists()] -def keymap_json_config(keyboard, keymap): +def keymap_json_config(keyboard, keymap, force_layout=None): """Extract keymap level config """ # TODO: resolve keymap.py and info.py circular dependencies from qmk.keymap import locate_keymap - keymap_folder = locate_keymap(keyboard, keymap).parent + keymap_folder = locate_keymap(keyboard, keymap, force_layout=force_layout).parent km_info_json = parse_configurator_json(keymap_folder / 'keymap.json') return km_info_json.get('config', {}) -def keymap_json(keyboard, keymap): +def keymap_json(keyboard, keymap, force_layout=None): """Generate the info.json data for a specific keymap. """ # TODO: resolve keymap.py and info.py circular dependencies from qmk.keymap import locate_keymap - keymap_folder = locate_keymap(keyboard, keymap).parent + keymap_folder = locate_keymap(keyboard, keymap, force_layout=force_layout).parent # Files to scan keymap_config = keymap_folder / 'config.h' @@ -1014,10 +1019,10 @@ def keymap_json(keyboard, keymap): keymap_file = keymap_folder / 'keymap.json' # Build the info.json file - kb_info_json = info_json(keyboard) + kb_info_json = info_json(keyboard, force_layout=force_layout) # Merge in the data from keymap.json - km_info_json = keymap_json_config(keyboard, keymap) if keymap_file.exists() else {} + km_info_json = keymap_json_config(keyboard, keymap, force_layout=force_layout) if keymap_file.exists() else {} deep_update(kb_info_json, km_info_json) # Merge in the data from config.h, and rules.mk diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index b7bf897377..91075f1bdc 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -420,7 +420,7 @@ def write(keymap_json): return write_file(keymap_file, keymap_content) -def locate_keymap(keyboard, keymap): +def locate_keymap(keyboard, keymap, force_layout=None): """Returns the path to a keymap for a specific keyboard. """ if not qmk.path.is_keyboard(keyboard): @@ -459,7 +459,7 @@ def locate_keymap(keyboard, keymap): return keymap_path # Check community layouts as a fallback - info = info_json(keyboard) + info = info_json(keyboard, force_layout=force_layout) community_parents = list(Path('layouts').glob('*/')) if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists(): diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index 2afb3033fc..baaf11eb34 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -1,11 +1,13 @@ """Functions for searching through QMK keyboards and keymaps. """ +from dataclasses import dataclass import contextlib import functools import fnmatch +import json import logging import re -from typing import Callable, List, Optional, Tuple +from typing import Callable, Dict, List, Optional, Tuple, Union from dotty_dict import dotty, Dotty from milc import cli @@ -15,7 +17,32 @@ from qmk.keyboard import list_keyboards, keyboard_folder from qmk.keymap import list_keymaps, locate_keymap from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget -TargetInfo = Tuple[str, str, dict] + +@dataclass +class KeyboardKeymapDesc: + keyboard: str + keymap: str + data: dict = None + extra_args: dict = None + + def __hash__(self) -> int: + return self.keyboard.__hash__() ^ self.keymap.__hash__() ^ json.dumps(self.extra_args, sort_keys=True).__hash__() + + def __lt__(self, other) -> bool: + return (self.keyboard, self.keymap, json.dumps(self.extra_args, sort_keys=True)) < (other.keyboard, other.keymap, json.dumps(other.extra_args, sort_keys=True)) + + def load_data(self): + data = keymap_json(self.keyboard, self.keymap) + self.data = data.to_dict() if isinstance(data, Dotty) else data + + @property + def dotty(self) -> Dotty: + return dotty(self.data) if self.data is not None else None + + def to_build_target(self) -> KeyboardKeymapBuildTarget: + target = KeyboardKeymapBuildTarget(keyboard=self.keyboard, keymap=self.keymap, json=self.data) + target.extra_args = self.extra_args + return target # by using a class for filters, we dont need to worry about capturing values @@ -36,7 +63,7 @@ class FilterFunction: value: Optional[str] func_name: str - apply: Callable[[TargetInfo], bool] + apply: Callable[[KeyboardKeymapDesc], bool] def __init__(self, key, value): self.key = key @@ -46,33 +73,29 @@ class FilterFunction: class Exists(FilterFunction): func_name = "exists" - def apply(self, target_info: TargetInfo) -> bool: - _kb, _km, info = target_info - return self.key in info + def apply(self, target_info: KeyboardKeymapDesc) -> bool: + return self.key in target_info.data class Absent(FilterFunction): func_name = "absent" - def apply(self, target_info: TargetInfo) -> bool: - _kb, _km, info = target_info - return self.key not in info + def apply(self, target_info: KeyboardKeymapDesc) -> bool: + return self.key not in target_info.data class Length(FilterFunction): func_name = "length" - def apply(self, target_info: TargetInfo) -> bool: - _kb, _km, info = target_info - return (self.key in info and len(info[self.key]) == int(self.value)) + def apply(self, target_info: KeyboardKeymapDesc) -> bool: + return (self.key in target_info.data and len(target_info.data[self.key]) == int(self.value)) class Contains(FilterFunction): func_name = "contains" - def apply(self, target_info: TargetInfo) -> bool: - _kb, _km, info = target_info - return (self.key in info and self.value in info[self.key]) + def apply(self, target_info: KeyboardKeymapDesc) -> bool: + return (self.key in target_info.data and self.value in target_info.data[self.key]) def _get_filter_class(func_name: str, key: str, value: str) -> Optional[FilterFunction]: @@ -109,12 +132,12 @@ def ignore_logging(): _set_log_level(old) -def _all_keymaps(keyboard): - """Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard. +def _all_keymaps(keyboard) -> List[KeyboardKeymapDesc]: + """Returns a list of KeyboardKeymapDesc for all keymaps for the given keyboard. """ with ignore_logging(): keyboard = keyboard_folder(keyboard) - return [(keyboard, keymap) for keymap in list_keymaps(keyboard)] + return [KeyboardKeymapDesc(keyboard, keymap) for keymap in list_keymaps(keyboard)] def _keymap_exists(keyboard, keymap): @@ -124,85 +147,91 @@ def _keymap_exists(keyboard, keymap): return keyboard if locate_keymap(keyboard, keymap) is not None else None -def _load_keymap_info(target: Tuple[str, str]) -> TargetInfo: - """Returns a tuple of (keyboard, keymap, info.json) for the given keyboard/keymap combination. +def _load_keymap_info(target: KeyboardKeymapDesc) -> KeyboardKeymapDesc: + """Ensures a KeyboardKeymapDesc has its data loaded. """ - kb, km = target with ignore_logging(): - return (kb, km, keymap_json(kb, km)) + target.load_data() # Ensure we load the data first + return target -def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]: - """Expand a list of make targets into a list of (keyboard, keymap) tuples. +def expand_make_targets(targets: List[Union[str, Tuple[str, Dict[str, str]]]]) -> List[KeyboardKeymapDesc]: + """Expand a list of make targets into a list of KeyboardKeymapDesc. Caters for 'all' in either keyboard or keymap, or both. """ split_targets = [] for target in targets: - split_target = target.split(':') + extra_args = None + if isinstance(target, tuple): + split_target = target[0].split(':') + extra_args = target[1] + else: + split_target = target.split(':') if len(split_target) != 2: cli.log.error(f"Invalid build target: {target}") return [] - split_targets.append((split_target[0], split_target[1])) + split_targets.append(KeyboardKeymapDesc(split_target[0], split_target[1], extra_args=extra_args)) return expand_keymap_targets(split_targets) -def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None) -> List[Tuple[str, str]]: - """Expand a keyboard input and keymap input into a list of (keyboard, keymap) tuples. +def _expand_keymap_target(target: KeyboardKeymapDesc, all_keyboards: List[str] = None) -> List[KeyboardKeymapDesc]: + """Expand a keyboard input and keymap input into a list of KeyboardKeymapDesc. Caters for 'all' in either keyboard or keymap, or both. """ if all_keyboards is None: all_keyboards = list_keyboards() - if keyboard == 'all': - if keymap == 'all': + if target.keyboard == 'all': + if target.keymap == 'all': cli.log.info('Retrieving list of all keyboards and keymaps...') targets = [] for kb in parallel_map(_all_keymaps, all_keyboards): targets.extend(kb) + for t in targets: + t.extra_args = target.extra_args return targets else: - cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...') - keyboard_filter = functools.partial(_keymap_exists, keymap=keymap) - return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))] + cli.log.info(f'Retrieving list of keyboards with keymap "{target.keymap}"...') + keyboard_filter = functools.partial(_keymap_exists, keymap=target.keymap) + return [KeyboardKeymapDesc(kb, target.keymap, extra_args=target.extra_args) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))] else: - if keymap == 'all': - cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...') - return _all_keymaps(keyboard) + if target.keymap == 'all': + cli.log.info(f'Retrieving list of keymaps for keyboard "{target.keyboard}"...') + targets = _all_keymaps(target.keyboard) + for t in targets: + t.extra_args = target.extra_args + return targets else: - return [(keyboard, keymap)] + return [target] -def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]: - """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples. +def expand_keymap_targets(targets: List[KeyboardKeymapDesc]) -> List[KeyboardKeymapDesc]: + """Expand a list of KeyboardKeymapDesc inclusive of 'all', into a list of explicit KeyboardKeymapDesc. """ overall_targets = [] all_keyboards = list_keyboards() for target in targets: - overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards)) + overall_targets.extend(_expand_keymap_target(target, all_keyboards)) return list(sorted(set(overall_targets))) -def _construct_build_target_kb_km(e): - return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1]) - +def _construct_build_target(e: KeyboardKeymapDesc): + return e.to_build_target() -def _construct_build_target_kb_km_json(e): - return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) - -def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]: - """Filter a list of (keyboard, keymap) tuples based on the supplied filters. +def _filter_keymap_targets(target_list: List[KeyboardKeymapDesc], filters: List[str] = []) -> List[KeyboardKeymapDesc]: + """Filter a list of KeyboardKeymapDesc based on the supplied filters. Optionally includes the values of the queried info.json keys. """ if len(filters) == 0: cli.log.info('Preparing target list...') - targets = list(set(parallel_map(_construct_build_target_kb_km, target_list))) + targets = target_list else: cli.log.info('Parsing data for all matching keyboard/keymap combinations...') - valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] + valid_targets = parallel_map(_load_keymap_info, target_list) function_re = re.compile(r'^(?P[a-zA-Z]+)\((?P[a-zA-Z0-9_\.]+)(,\s*(?P[^#]+))?\)$') equals_re = re.compile(r'^(?P[a-zA-Z0-9_\.]+)\s*=\s*(?P[^#]+)$') @@ -220,7 +249,7 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str if filter_class is None: cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}') continue - valid_keymaps = filter(filter_class.apply, valid_keymaps) + valid_targets = filter(filter_class.apply, valid_targets) value_str = f", {{fg_cyan}}{value}{{fg_reset}}" if value is not None else "" cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}{value_str})...') @@ -234,32 +263,42 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str expr = fnmatch.translate(v) rule = re.compile(f'^{expr}$', re.IGNORECASE) - def f(e): - lhs = e[2].get(k) + def f(e: KeyboardKeymapDesc): + lhs = e.dotty.get(k) lhs = str(False if lhs is None else lhs) return rule.search(lhs) is not None return f - valid_keymaps = filter(_make_filter(key, value), valid_keymaps) + valid_targets = filter(_make_filter(key, value), valid_targets) else: cli.log.warning(f'Unrecognized filter expression: {filter_expr}') continue cli.log.info('Preparing target list...') - valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps] # need to convert dotty_dict back to dict because it doesn't survive parallelisation - targets = list(set(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps)))) + targets = list(sorted(set(valid_targets))) return targets -def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]: +def search_keymap_targets(targets: List[Union[Tuple[str, str], Tuple[str, str, Dict[str, str]]]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]: """Search for build targets matching the supplied criteria. """ - return _filter_keymap_targets(expand_keymap_targets(targets), filters) + def _make_desc(e): + if len(e) == 3: + return KeyboardKeymapDesc(keyboard=e[0], keymap=e[1], extra_args=e[2]) + else: + return KeyboardKeymapDesc(keyboard=e[0], keymap=e[1]) + + targets = map(_make_desc, targets) + targets = _filter_keymap_targets(expand_keymap_targets(targets), filters) + targets = list(set(parallel_map(_construct_build_target, list(targets)))) + return sorted(targets) -def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]: +def search_make_targets(targets: List[Union[str, Tuple[str, Dict[str, str]]]], filters: List[str] = []) -> List[BuildTarget]: """Search for build targets matching the supplied criteria. """ - return _filter_keymap_targets(expand_make_targets(targets), filters) + targets = _filter_keymap_targets(expand_make_targets(targets), filters) + targets = list(set(parallel_map(_construct_build_target, list(targets)))) + return sorted(targets) diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py index 1e5823b229..1c2a97f9c1 100644 --- a/lib/python/qmk/userspace.py +++ b/lib/python/qmk/userspace.py @@ -1,4 +1,4 @@ -# Copyright 2023 Nick Brassel (@tzarc) +# Copyright 2023-2024 Nick Brassel (@tzarc) # SPDX-License-Identifier: GPL-2.0-or-later from os import environ from pathlib import Path @@ -77,31 +77,43 @@ class UserspaceDefs: raise exception # Iterate through each version of the schema, starting with the latest and decreasing to v1 - try: - validate(json, 'qmk.user_repo.v1') - self.__load_v1(json) - success = True - except jsonschema.ValidationError as err: - exception.add('qmk.user_repo.v1', err) + schema_versions = [ + ('qmk.user_repo.v1_1', self.__load_v1_1), # + ('qmk.user_repo.v1', self.__load_v1) # + ] + + for v in schema_versions: + schema = v[0] + loader = v[1] + try: + validate(json, schema) + loader(json) + success = True + break + except jsonschema.ValidationError as err: + exception.add(schema, err) if not success: raise exception def save(self): target_json = { - "userspace_version": "1.0", # Needs to match latest version + "userspace_version": "1.1", # Needs to match latest version "build_targets": [] } for e in self.build_targets: if isinstance(e, dict): - target_json['build_targets'].append([e['keyboard'], e['keymap']]) + entry = [e['keyboard'], e['keymap']] + if 'env' in e: + entry.append(e['env']) + target_json['build_targets'].append(entry) elif isinstance(e, Path): target_json['build_targets'].append(str(e.relative_to(self.path.parent))) try: # Ensure what we're writing validates against the latest version of the schema - validate(target_json, 'qmk.user_repo.v1') + validate(target_json, 'qmk.user_repo.v1_1') except jsonschema.ValidationError as err: cli.log.error(f'Could not save userspace file: {err}') return False @@ -114,7 +126,7 @@ class UserspaceDefs: cli.log.info(f'Saved userspace file to {self.path}.') return True - def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): + def add_target(self, keyboard=None, keymap=None, build_env=None, json_path=None, do_print=True): if json_path is not None: # Assume we're adding a json filename/path json_path = Path(json_path) @@ -128,6 +140,8 @@ class UserspaceDefs: elif keyboard is not None and keymap is not None: # Both keyboard/keymap specified e = {"keyboard": keyboard, "keymap": keymap} + if build_env is not None: + e['env'] = build_env if e not in self.build_targets: self.build_targets.append(e) if do_print: @@ -136,7 +150,7 @@ class UserspaceDefs: if do_print: cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.') - def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): + def remove_target(self, keyboard=None, keymap=None, build_env=None, json_path=None, do_print=True): if json_path is not None: # Assume we're removing a json filename/path json_path = Path(json_path) @@ -150,6 +164,8 @@ class UserspaceDefs: elif keyboard is not None and keymap is not None: # Both keyboard/keymap specified e = {"keyboard": keyboard, "keymap": keymap} + if build_env is not None: + e['env'] = build_env if e in self.build_targets: self.build_targets.remove(e) if do_print: @@ -160,12 +176,26 @@ class UserspaceDefs: def __load_v1(self, json): for e in json['build_targets']: - if isinstance(e, list) and len(e) == 2: - self.add_target(keyboard=e[0], keymap=e[1], do_print=False) - if isinstance(e, str): - p = self.path.parent / e - if p.exists() and p.suffix == '.json': - self.add_target(json_path=p, do_print=False) + self.__load_v1_target(e) + + def __load_v1_1(self, json): + for e in json['build_targets']: + self.__load_v1_1_target(e) + + def __load_v1_target(self, e): + if isinstance(e, list) and len(e) == 2: + self.add_target(keyboard=e[0], keymap=e[1], do_print=False) + if isinstance(e, str): + p = self.path.parent / e + if p.exists() and p.suffix == '.json': + self.add_target(json_path=p, do_print=False) + + def __load_v1_1_target(self, e): + # v1.1 adds support for a third item in the build target tuple; kvp's for environment + if isinstance(e, list) and len(e) == 3: + self.add_target(keyboard=e[0], keymap=e[1], build_env=e[2], do_print=False) + else: + self.__load_v1_target(e) class UserspaceValidationError(Exception): -- cgit v1.2.3 From 39324e642f000c86a46d1974651e7a474d9375d8 Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Thu, 15 Aug 2024 14:37:27 +0100 Subject: Update Discord URL (#24283) --- lib/python/qmk/cli/doctor/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py index dd8b58b2c7..5215d3a7ee 100755 --- a/lib/python/qmk/cli/doctor/main.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -187,5 +187,5 @@ def doctor(cli): return 1 else: cli.log.info('{fg_red}Major problems detected, please fix these problems before proceeding.') - cli.log.info('{fg_blue}Check out the FAQ (https://docs.qmk.fm/#/faq_build) or join the QMK Discord (https://discord.gg/Uq7gcHh) for help.') + cli.log.info('{fg_blue}Check out the FAQ (https://docs.qmk.fm/#/faq_build) or join the QMK Discord (https://discord.gg/qmk) for help.') return 2 -- cgit v1.2.3 From 5741eb7b10ecc7da4453424004fd291ed0573e4d Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Mon, 26 Aug 2024 09:57:35 +1000 Subject: Fixup python tests for missing `via` keymaps. --- lib/python/qmk/tests/test_cli_commands.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index 2b3d6f4049..4c322e0c9d 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -118,21 +118,18 @@ def test_list_keymaps_kb_only(): result = check_subcommand('list-keymaps', '-kb', 'contra') check_returncode(result) assert 'default' in result.stdout - assert 'via' in result.stdout def test_list_keymaps_vendor_kb(): result = check_subcommand('list-keymaps', '-kb', 'ai03/lunar') check_returncode(result) assert 'default' in result.stdout - assert 'via' in result.stdout def test_list_keymaps_vendor_kb_rev(): result = check_subcommand('list-keymaps', '-kb', 'kbdfans/kbd67/mkiirgb/v2') check_returncode(result) assert 'default' in result.stdout - assert 'via' in result.stdout def test_list_keymaps_no_keyboard_found(): -- cgit v1.2.3 From 7a4f21d34086278cf1484e4e92837260f76b1396 Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Mon, 26 Aug 2024 03:56:03 +0100 Subject: Reject via keymaps in lint (#24325) --- lib/python/qmk/cli/lint.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib/python') diff --git a/lib/python/qmk/cli/lint.py b/lib/python/qmk/cli/lint.py index ba0c3f274c..efb29704ae 100644 --- a/lib/python/qmk/cli/lint.py +++ b/lib/python/qmk/cli/lint.py @@ -14,15 +14,16 @@ from qmk.c_parse import c_source_files CHIBIOS_CONF_CHECKS = ['chconf.h', 'halconf.h', 'mcuconf.h', 'board.h'] INVALID_KB_FEATURES = set(['encoder_map', 'dip_switch_map', 'combo', 'tap_dance', 'via']) +INVALID_KM_NAMES = ['via', 'vial'] def _list_defaultish_keymaps(kb): """Return default like keymaps for a given keyboard """ - defaultish = ['ansi', 'iso', 'via'] + defaultish = ['ansi', 'iso'] # This is only here to flag it as "testable", so it doesn't fly under the radar during PR - defaultish.append('vial') + defaultish.extend(INVALID_KM_NAMES) keymaps = set() for x in list_keymaps(kb): @@ -136,6 +137,11 @@ def keymap_check(kb, km): cli.log.error("%s: Can't find %s keymap.", kb, km) return ok + if km in INVALID_KM_NAMES: + ok = False + cli.log.error("%s: The keymap %s should not exist!", kb, km) + return ok + # Additional checks invalid_files = git_get_ignored_files(keymap_path.parent.as_posix()) for file in invalid_files: -- cgit v1.2.3