{ "corpus": "corpus/codeflash.json", "arms": [ "hypothesis", "mutations", "llm-plan", "breakwater" ], "provider": "tokenfactory", "backend": "contree", "items": [ { "id": "Math-Interpreter#1", "pr": 1, "title": "⚡️ Speed up method `Lexer.generate_number` by 11%", "base": "c7c1b7e931e873f6d919522b50911ee3571b2e1e", "head": "04c05ed37563f0158b9148535c69390e3464dafd", "merged": "2025-02-09", "after_ultra_release": false, "known": [], "repo": "https://github.com/hoopinwhoopin/Math-Interpreter", "install": { "mode": "src", "subdir": "", "deps": [], "exclude_newer": "2025-02-10" }, "targets": [ { "target": "lexer:Lexer#generate_tokens", "run_id": "bt-Math-Interpreter#1-1d8136", "image": "6cbae808-5ef0-4683-8481-a6120ba99109", "backend": "contree", "usd": 0.0456, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 7.8, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 5.7, "error": null }, "breakwater": { "classes": [ "P2|outcome|exc:builtins.Exception|ok|long-digits+non-ascii", "P2|outcome|exc:builtins.Exception|ok|non-ascii", "P2|outcome|exc:builtins.Exception|ok|non-ascii+whitespace" ], "known": [], "seconds": 45.0, "error": null }, "llm-plan": { "classes": [ "P2|outcome|exc:builtins.Exception|ok|non-ascii" ], "known": [], "seconds": 4.8, "error": null } }, "findings": [ { "signature": "P2|outcome|exc:builtins.Exception|ok|non-ascii", "probe": "P2", "detail": "outcome", "args": [ [ "1١" ] ], "base": { "status": "exc", "exc": "builtins.Exception" }, "head": { "status": "ok", "obs": { "value": [ "NUMBER11.0" ] } }, "found_by": [ "seed", "mutation:equivalents", "mutation:separators", "mutation:signs" ] }, { "signature": "P2|outcome|exc:builtins.Exception|ok|long-digits+non-ascii", "probe": "P2", "detail": "outcome", "args": [ [ "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000١" ] ], "base": { "status": "exc", "exc": "builtins.Exception" }, "head": { "status": "ok", "obs": { "value": [ "NUMBERinf" ] } }, "found_by": [ "mutation:huge-number" ] }, { "signature": "P2|outcome|exc:builtins.Exception|ok|non-ascii+whitespace", "probe": "P2", "detail": "outcome", "args": [ [ "1١ " ] ], "base": { "status": "exc", "exc": "builtins.Exception" }, "head": { "status": "ok", "obs": { "value": [ "NUMBER11.0" ] } }, "found_by": [ "mutation:whitespace-inner", "mutation:whitespace-outer" ] } ], "perf": { "slower_families": [ { "id": "R1", "family": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many sh", "verdict": "slower", "ladder": [ 0.9763257852045161, 1.010942405039847, 1.0433034274148216, 1.0527257417148235, 1.0549324662912727, 1.051966573846875, 1.0507427821552848 ], "origin": "archive" }, { "id": "R2", "family": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allo", "verdict": "slower", "ladder": [ 0.9764166975793878, 1.0107375575993678, 1.042012520049912, 1.0510971844516417, 1.0532212409412909, 1.050326452184568, 1.0491498485921533 ], "origin": "archive" }, { "id": "F5", "family": "Many one‑digit numbers separated by operators", "verdict": "slower", "ladder": [ 0.9761670122449891, 1.0103584762292424, 1.0418301658705715, 1.0510654639526884, 1.0532090642238103, 1.0503245612811982, 1.0491491530412056 ], "origin": "plan" }, { "id": "R3", "family": "A sequence of n single‑digit numbers separated by ‘+’ (e.g., \"1+2+3+…\").", "verdict": "slower", "ladder": [ 0.9761670122449891, 1.0103584762292424, 1.0418301658705715, 1.0510654639526884, 1.0532090642238103, 1.0503245612811982, 1.0491491530412056 ], "origin": "archive" } ], "worst_valid_ratio": 1.0549324662912727, "worst_family": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many sh", "speedup_ratio_small": 0.8766084890590644, "report": { "run_id": "bt-Math-Interpreter#1-1d8136", "target": "lexer:Lexer#generate_tokens", "image": "6cbae808-5ef0-4683-8481-a6120ba99109", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "single very long numeric literal", "why": "The head version builds the number using a list + append + join (O(n)) while the base version concatenates strings in a loop (O(n²)). For long numbers the head should be dramatically faster.", "size_meaning": "n = number of digit characters in the literal", "make_source": "def make(n):\n # one token: a long integer like \"111...1\" (n digits)\n text = '1' * n\n return [[text]]", "example_args": [ [ "1111111111" ] ], "origin": "plan", "screen_ratio": 0.922662306826316, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "Long integer literals (many digits)", "why": "Base used repeated string concatenation `number_str += ch` inside the digit loop, which is O(L²) for a number of length L. Head switched to `list.append` + final `''.join`, giving O(L) time. The speedup grows with the number of digits.", "size_meaning": "Number of digits in the integer literal", "make_source": "def make(n):\n # n digits, e.g. n=1000 -> '111...1'\n s = '1' * max(1, n)\n return [[s]]", "example_args": [ [ 1 ], [ 11 ], [ 1111111111 ] ], "origin": "plan", "screen_ratio": 0.922662306826316, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "Long numeric literals", "why": "The new version builds the number text in a list and joins once, which is asymptotically faster than repeated string concatenation for long numbers.", "size_meaning": "n = number of digits in the literal", "make_source": "def make(n):\n # single very long integer literal, e.g. \"111...1\" (n digits)\n return [[\"1\" * n]]", "example_args": [ [ "1" ], [ "11" ], [ "111" ], [ "1111" ] ], "origin": "plan", "screen_ratio": 0.922662306826316, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "many tiny numbers (single digit) separated by operators/whitespace", "why": "Each call to generate_number creates a new list and later joins it. For a one‑character number the list allocation + join overhead outweighs the cheap string concatenation the base version does, so the head can be slower when the input consists of a huge quantity of one‑digit numbers.", "size_meaning": "n = how many single‑digit numbers appear in the expression", "make_source": "def make(n):\n # pattern: \"1+2+3+...\" (n numbers, n‑1 plus signs)\n parts = []\n for i in range(n):\n parts.append(str((i % 9) + 1)) # digits 1‑9\n if i != n-1:\n parts.append('+')\n text = ''.join(parts)\n return [[text]]", "example_args": [ [ "1+2+3+4+5" ] ], "origin": "plan", "screen_ratio": 1.0416193654965378, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "Many one‑digit numbers separated by operators", "why": "For each one‑digit number the head creates a list, appends one character, then calls `''.join`. The allocation + join overhead dominates the trivial concatenation the base does (`number_str += ch`). When the input contains O(n) such numbers the extra per‑number work makes the head slower.", "size_meaning": "Count of single‑digit numbers (each followed by a '+')", "make_source": "def make(n):\n # produce '1+2+3+...+' with n numbers\n parts = []\n for i in range(1, n+1):\n parts.append(str(i % 10))\n if i != n:\n parts.append('+')\n s = ''.join(parts)\n return [[s]]", "example_args": [ [ 1 ] ], "origin": "plan", "screen_ratio": 1.0418301658705715, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "Many single‑digit numbers separated by operators", "why": "For each one‑digit number the new code creates a list, appends one character and then calls `''.join`. The allocation/join overhead dominates the tiny work, while the old version just did `s += ch` which is cheap for length‑1 strings.", "size_meaning": "n = how many numbers (≈ 2 n characters in the source)", "make_source": "def make(n):\n # produce \"1+2+3+...\" with n numbers\n parts = []\n for i in range(n):\n parts.append(str((i % 9) + 1)) # digit 1‑9\n if i != n - 1:\n parts.append('+')\n return [[''.join(parts)]]", "example_args": [ [ "1" ], [ "1+2" ], [ "1+2+3" ], [ "1+2+3+4" ] ], "origin": "plan", "screen_ratio": 1.0416193654965378, "valid_fraction": 1.0, "error": null }, { "id": "R1", "kind": "suspected_slowdown", "family": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many sh", "why": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many short numbers that typical expressions contain (e.g., \"1+2+3+…\").", "size_meaning": "Number of single‑digit numbers (each separated by a '+') – total input length ≈ 2·n.", "make_source": "def make(n):\n # n copies of the digit '1' separated by '+'\n s = '+'.join(['1'] * n)\n return [[s]]", "example_args": [ [ 1 ] ], "origin": "archive", "screen_ratio": 1.0433034274148216, "valid_fraction": 1.0, "error": null }, { "id": "R2", "kind": "suspected_slowdown", "family": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allo", "why": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allocation overhead dominates, making the new code slower than the old simple `+=` on a one‑character string.", "size_meaning": "Count of single‑digit numbers separated by '+' operators.", "make_source": "def make(n):\n # Build \"1+2+3+…\" with n numbers (the last number has no trailing '+').\n parts = []\n for i in range(n):\n parts.append(str(i % 10))\n if i != n - 1:\n parts.append('+')\n s = ''.join(parts)\n return [[s]]", "example_args": [], "origin": "archive", "screen_ratio": 1.042012520049912, "valid_fraction": 1.0, "error": null }, { "id": "R3", "kind": "suspected_slowdown", "family": "A sequence of n single‑digit numbers separated by ‘+’ (e.g., \"1+2+3+…\").", "why": "Each digit causes a separate call to generate_number that processes only one character. The head version now allocates a list, appends one character, and calls ''.join, which adds allocation overhead compared to the base version's trivial one‑character string concatenation. With many such calls the extra per‑call cost dominates.", "size_meaning": "Number of single‑digit numbers (≈ number of tokens).", "make_source": "def make(n):\n # build \"1+2+3+...\" up to n numbers\n parts = []\n for i in range(1, n+1):\n parts.append(str(i % 10))\n if i != n:\n parts.append('+')\n s = ''.join(parts)\n return [[s]]", "example_args": [ [ "1+2+3+4+5" ] ], "origin": "archive", "screen_ratio": 1.0418301658705715, "valid_fraction": 1.0, "error": null }, { "id": "R4", "kind": "suspected_slowdown", "family": "Digits vary but remain single-digit", "why": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many short numbers that typical expressions contain (e.g., \"1+2+3+…\").", "size_meaning": "Number of single‑digit numbers (each separated by a '+') – total input length ≈ 2·n.", "make_source": "def make(n):\n digits = [str(random.randint(0, 9)) for _ in range(n)]\n s = '+'.join(digits)\n return [[s]]", "example_args": [ [ 1 ] ], "origin": "archive", "screen_ratio": 1.0413330251689885, "valid_fraction": 1.0, "error": null }, { "id": "R1.1", "kind": "suspected_slowdown", "family": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. Overhead from list and join dominates", "why": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many short numbers that typical expressions contain (e.g., \"1+2+3+…\").", "size_meaning": "Number of single‑digit numbers (each separated by a '+') – total input length ≈ 2·n.", "make_source": "def make(n):\n return [[\"+\".join([str(i % 10)] * n))]", "example_args": [ [ 1 ] ], "origin": "expand:R1", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: syntax error: closing parenthesis ')' does not match opening parenthesis '[' (line 3)" }, { "id": "R1.2", "kind": "suspected_slowdown", "family": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. Overhead from list and join dominates", "why": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many short numbers that typical expressions contain (e.g., \"1+2+3+…\").", "size_meaning": "Number of single‑digit numbers (each separated by a '+') – total input length ≈ 2·n.", "make_source": "def make(n):\n s = '-'.join(['5'] * n)\n return [[s]]", "example_args": [ [ 1 ] ], "origin": "expand:R1", "screen_ratio": 1.0407913207425334, "valid_fraction": 1.0, "error": null }, { "id": "R1.3", "kind": "suspected_slowdown", "family": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. Overhead from list and join dominates", "why": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many short numbers that typical expressions contain (e.g., \"1+2+3+…\").", "size_meaning": "Number of single‑digit numbers (each separated by a '+') – total input length ≈ 2·n.", "make_source": "def make(n):\n return [[('0+' * n).strip('+')]}]}, {", "example_args": [ [ 1 ] ], "origin": "expand:R1", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: syntax error: closing parenthesis '}' does not match opening parenthesis '[' (line 3)" }, { "id": "R2.1", "kind": "suspected_slowdown", "family": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allocation overhead dominates, making the new code slower than the old simple `+=` on a one‑character string.", "why": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allocation overhead dominates, making the new code slower than the old simple `+=` on a one‑character string.", "size_meaning": "Count of single‑digit numbers separated by '+' operators.", "make_source": "\ndef make(n):\n \\\"\\\"\\\"Generate expression of alternating digits and addition signs starting with digit:\\\n Result pattern: \"0+1+2+...+(n-1)\"\n Value comes from repeating cycle of characters [\"0\", \"+\", \"1\", \"+\", ...]\n Takes first 2*n-1 elements.\n \\\"\\\"\\\"\n cyc = ['0', '+']\n seq = []\n for i in range(n):\n seq.append(str(i))\n if i < n - 1:\n seq.append('+')\n expr = ''.join(seq)\n return [[expr]]", "example_args": [], "origin": "expand:R2", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: syntax error: unexpected character after line continuation character (line 4)" }, { "id": "R2.2", "kind": "suspected_slowdown", "family": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allocation overhead dominates, making the new code slower than the old simple `+=` on a one‑character string.", "why": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allocation overhead dominates, making the new code slower than the old simple `+=` on a one‑character string.", "size_meaning": "Count of single‑digit numbers separated by '+' operators.", "make_source": "import itertools as _\n\ndef make(n):\n token_str = ''.join(_.chain.from_iterable((str(d), '+') for d in range(n))[:-1])\n return [[token_str]]", "example_args": [], "origin": "expand:R2", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: imports are not allowed" } ], "findings": [ { "family_id": "R1", "kind": "suspected_slowdown", "family": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many sh", "why": "Each call to generate_number now creates a list and calls ''.join for a one‑character number. The overhead of list allocation and join dominates for the many short numbers that typical expressions contain (e.g., \"1+2+3+…\").", "origin": "archive", "screen_ratio": 1.0433034274148216, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 29990.435, "head_ir": 29280.435, "input_chars": 7.0, "valid_fraction": 1.0, "ratio": 0.9763257852045161, "judged": true }, { "n": 4, "base_ir": 52440.4825, "head_ir": 53014.3075, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.010942405039847, "judged": true }, { "n": 16, "base_ir": 188466.3175, "head_ir": 196627.555, "input_chars": 35.0, "valid_fraction": 1.0, "ratio": 1.0433034274148216, "judged": true }, { "n": 64, "base_ir": 732337.73, "head_ir": 770950.78, "input_chars": 131.0, "valid_fraction": 1.0, "ratio": 1.0527257417148235, "judged": true }, { "n": 256, "base_ir": 2918962.3815789474, "head_ir": 3079308.1842105263, "input_chars": 515.0, "valid_fraction": 1.0, "ratio": 1.0549324662912727, "judged": true }, { "n": 1024, "base_ir": 12251477.944444444, "head_ir": 12888145.277777778, "input_chars": 2051.0, "valid_fraction": 1.0, "ratio": 1.051966573846875, "judged": true }, { "n": 4096, "base_ir": 50492176.25, "head_ir": 53054289.75, "input_chars": 8195.0, "valid_fraction": 1.0, "ratio": 1.0507427821552848, "judged": true } ], "slope_base": 1.0442702061172198, "slope_head": 1.0525507574653248, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.0549324662912727, "worst_search": null }, { "family_id": "R2", "kind": "suspected_slowdown", "family": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allo", "why": "Each call to `generate_number` now creates a Python list, appends a single character, and then joins it. For a huge sequence of tiny numbers the per‑number allocation overhead dominates, making the new code slower than the old simple `+=` on a one‑character string.", "origin": "archive", "screen_ratio": 1.042012520049912, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 29936.435, "head_ir": 29230.435, "input_chars": 7.0, "valid_fraction": 1.0, "ratio": 0.9764166975793878, "judged": true }, { "n": 4, "base_ir": 52416.4825, "head_ir": 52979.3075, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.0107375575993678, "judged": true }, { "n": 16, "base_ir": 188830.3175, "head_ir": 196763.555, "input_chars": 35.0, "valid_fraction": 1.0, "ratio": 1.042012520049912, "judged": true }, { "n": 64, "base_ir": 734209.73, "head_ir": 771725.78, "input_chars": 131.0, "valid_fraction": 1.0, "ratio": 1.0510971844516417, "judged": true }, { "n": 256, "base_ir": 2926910.3815789474, "head_ir": 3082684.1842105263, "input_chars": 515.0, "valid_fraction": 1.0, "ratio": 1.0532212409412909, "judged": true }, { "n": 1024, "base_ir": 12283685.944444444, "head_ir": 12901880.277777778, "input_chars": 2051.0, "valid_fraction": 1.0, "ratio": 1.050326452184568, "judged": true }, { "n": 4096, "base_ir": 50621468.25, "head_ir": 53109505.75, "input_chars": 8195.0, "valid_fraction": 1.0, "ratio": 1.0491498485921533, "judged": true } ], "slope_base": 1.044813311565244, "slope_head": 1.052876717052794, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.0532212409412909, "worst_search": null }, { "family_id": "F5", "kind": "suspected_slowdown", "family": "Many one‑digit numbers separated by operators", "why": "For each one‑digit number the head creates a list, appends one character, then calls `''.join`. The allocation + join overhead dominates the trivial concatenation the base does (`number_str += ch`). When the input contains O(n) such numbers the extra per‑number work makes the head slower.", "origin": "plan", "screen_ratio": 1.0418301658705715, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 30000.435, "head_ir": 29285.435, "input_chars": 7.0, "valid_fraction": 1.0, "ratio": 0.9761670122449891, "judged": true }, { "n": 4, "base_ir": 52500.4825, "head_ir": 53044.3075, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.0103584762292424, "judged": true }, { "n": 16, "base_ir": 188936.3175, "head_ir": 196839.555, "input_chars": 35.0, "valid_fraction": 1.0, "ratio": 1.0418301658705715, "judged": true }, { "n": 64, "base_ir": 734293.73, "head_ir": 771790.78, "input_chars": 131.0, "valid_fraction": 1.0, "ratio": 1.0510654639526884, "judged": true }, { "n": 256, "base_ir": 2927016.3815789474, "head_ir": 3082760.1842105263, "input_chars": 515.0, "valid_fraction": 1.0, "ratio": 1.0532090642238103, "judged": true }, { "n": 1024, "base_ir": 12283769.944444444, "head_ir": 12901945.277777778, "input_chars": 2051.0, "valid_fraction": 1.0, "ratio": 1.0503245612811982, "judged": true }, { "n": 4096, "base_ir": 50621574.25, "head_ir": 53109581.75, "input_chars": 8195.0, "valid_fraction": 1.0, "ratio": 1.0491491530412056, "judged": true } ], "slope_base": 1.0445353876550403, "slope_head": 1.0526473395483273, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.0532090642238103, "worst_search": null }, { "family_id": "R3", "kind": "suspected_slowdown", "family": "A sequence of n single‑digit numbers separated by ‘+’ (e.g., \"1+2+3+…\").", "why": "Each digit causes a separate call to generate_number that processes only one character. The head version now allocates a list, appends one character, and calls ''.join, which adds allocation overhead compared to the base version's trivial one‑character string concatenation. With many such calls the extra per‑call cost dominates.", "origin": "archive", "screen_ratio": 1.0418301658705715, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 30000.435, "head_ir": 29285.435, "input_chars": 7.0, "valid_fraction": 1.0, "ratio": 0.9761670122449891, "judged": true }, { "n": 4, "base_ir": 52500.4825, "head_ir": 53044.3075, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.0103584762292424, "judged": true }, { "n": 16, "base_ir": 188936.3175, "head_ir": 196839.555, "input_chars": 35.0, "valid_fraction": 1.0, "ratio": 1.0418301658705715, "judged": true }, { "n": 64, "base_ir": 734293.73, "head_ir": 771790.78, "input_chars": 131.0, "valid_fraction": 1.0, "ratio": 1.0510654639526884, "judged": true }, { "n": 256, "base_ir": 2927016.3815789474, "head_ir": 3082760.1842105263, "input_chars": 515.0, "valid_fraction": 1.0, "ratio": 1.0532090642238103, "judged": true }, { "n": 1024, "base_ir": 12283769.944444444, "head_ir": 12901945.277777778, "input_chars": 2051.0, "valid_fraction": 1.0, "ratio": 1.0503245612811982, "judged": true }, { "n": 4096, "base_ir": 50621574.25, "head_ir": 53109581.75, "input_chars": 8195.0, "valid_fraction": 1.0, "ratio": 1.0491491530412056, "judged": true } ], "slope_base": 1.0445353876550403, "slope_head": 1.0526473395483273, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.0532090642238103, "worst_search": null }, { "family_id": "F1", "kind": "claimed_speedup", "family": "single very long numeric literal", "why": "The head version builds the number using a list + append + join (O(n)) while the base version concatenates strings in a loop (O(n²)). For long numbers the head should be dramatically faster.", "origin": "plan", "screen_ratio": 0.922662306826316, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 20648.045, "head_ir": 19302.045, "input_chars": 6.0, "valid_fraction": 1.0, "ratio": 0.9348122304072856, "judged": true }, { "n": 4, "base_ir": 23948.045, "head_ir": 22232.045, "input_chars": 8.0, "valid_fraction": 1.0, "ratio": 0.9283448815968067, "judged": true }, { "n": 16, "base_ir": 45075.045, "head_ir": 41589.045, "input_chars": 20.0, "valid_fraction": 1.0, "ratio": 0.922662306826316, "judged": true }, { "n": 64, "base_ir": 127077.045, "head_ir": 115942.045, "input_chars": 68.0, "valid_fraction": 1.0, "ratio": 0.9123759920605645, "judged": true }, { "n": 256, "base_ir": 447152.92763157893, "head_ir": 403614.92763157893, "input_chars": 260.0, "valid_fraction": 1.0, "ratio": 0.9026328638155041, "judged": true }, { "n": 1024, "base_ir": 1730205.394736842, "head_ir": 1516712.7368421052, "input_chars": 1028.0, "valid_fraction": 1.0, "ratio": 0.8766084890590644, "judged": true }, { "n": 4096, "base_ir": 6935094.75, "head_ir": 6125963.625, "input_chars": 4100.0, "valid_fraction": 1.0, "ratio": 0.8833280360012384, "judged": true } ], "slope_base": 0.8962148392989655, "slope_head": 0.8866521489110624, "top_inputs": [], "error": null, "verdict": "faster", "worst_ratio": 0.9348122304072856, "worst_search": null } ], "seconds": { "plan": 9.98, "screen": 172.36, "full": 205.48, "total": 387.82 } } } } ] }, { "id": "micropie#3", "pr": 3, "title": "⚡️ Speed up method `Server._parse_cookies` by 6%", "base": "b760304db8ecfe6577a17fac0623c07972427516", "head": "19ce2334b6c8ce9217d0e7b7aea3a66d75f83b19", "merged": "2025-02-03", "after_ultra_release": false, "known": [], "repo": "https://github.com/hoopinwhoopin/micropie", "install": { "exclude_newer": "2025-02-04" }, "targets": [ { "target": "MicroPie:App()#_parse_cookies", "run_id": "bt-micropie#3-ae94a9", "image": "f05e47b2-dfb5-4e4e-8bff-cb2a2dcddd6b", "backend": "contree", "usd": 0.0535, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 10.1, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 6.6, "error": null }, "breakwater": { "classes": [], "known": [], "seconds": 26.7, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 6.0, "error": null } }, "findings": [], "perf": { "slower_families": [ { "id": "F5", "family": "many_empty_segments", "verdict": "slower", "ladder": [ 1.0262405855229926, 1.0364046320534492, 1.0950635955029355, 1.2440709416633968, 1.456898020929322, 1.604931540857205, 1.6035186029976054 ], "origin": "plan" }, { "id": "F4", "family": "many_semicolon_segments_without_equals", "verdict": "slower", "ladder": [ 1.016924586328987, 1.0281007565691043, 1.0846397547306883, 1.2003535964039693, 1.3175636101318646, 1.3718171795783076, 1.3727797324019853 ], "origin": "plan" }, { "id": "F6", "family": "malformed cookie header – many semicolon‑separated parts that do NOT contain \"=\"", "verdict": "slower", "ladder": [ 1.01691611608539, 1.028074672488551, 1.0843783653713397, 1.1982828403826216, 1.3116162621500775, 1.3662352005134333, 1.3619241635701043 ], "origin": "plan" }, { "id": "F4.1", "family": "many_semicolon_segments_with_plus_in_middle", "verdict": "slower", "ladder": [ 1.016907251585494, 1.0276886347772645, 1.0842533733254556, 1.1959682743253186, 1.2980715190680983, 1.3475857001204052, 1.343159033208263 ], "origin": "expand:F4" } ], "worst_valid_ratio": 1.604931540857205, "worst_family": "many_empty_segments", "speedup_ratio_small": 1.0056459013831607, "report": { "run_id": "bt-micropie#3-ae94a9", "target": "MicroPie:App()#_parse_cookies", "image": "f05e47b2-dfb5-4e4e-8bff-cb2a2dcddd6b", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "single_cookie_with_equals", "why": "The change was introduced with a docstring and a tiny refactor; the intended fast path is the common case of a short Cookie header containing a few well‑formed \"name=value\" pairs. Both versions allocate a dict once and strip only the parts that contain \"=\", so the new code should not be slower here.", "size_meaning": "Number of cookies in the header (each cookie adds a \"k=v;\" segment).", "make_source": "def make(n):\n # n cookies \"c0=v0;c1=v1;...\"\n parts = [f'c{i}=v{i}' for i in range(n)]\n header = ';'.join(parts)\n return [[header]]", "example_args": [], "origin": "plan", "screen_ratio": 1.009192950127595, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "many_well_formed_cookies", "why": "Typical cookie headers contain many key=value pairs. The change moves the strip outside the equality test, which the author likely expected to avoid repeated attribute look‑ups and make the hot path tighter.", "size_meaning": "Number of cookies (each ≈ 10 bytes).", "make_source": "def make(n):\n # generate a header with n cookies like \"c0=v0; c1=v1; ...\"\n parts = [f\"c{i}=v{i}\" for i in range(n)]\n header = \"; \".join(parts)\n return [[header]]", "example_args": [], "origin": "plan", "screen_ratio": 1.0072629307261658, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "well‑formed cookie header – each part contains a \"=\" and no surrounding whitespace", "why": "The new code strips each part before the \"=\" test; when every part already has an \"=\" and no spaces the extra strip is a no‑op, so the fast‑path of the original (strip‑then‑split) and the new (strip‑then‑split) take the same time. The change was advertised as a cleanup that should not hurt this common case.", "size_meaning": "n = number of cookie key/value pairs", "make_source": "def make(n):\n # Build a header like \"k0=v0;k1=v1;...;k{n-1}=v{n-1}\"\n header = \";\".join(f\"k{i}=v{i}\" for i in range(n))\n return [[header]]", "example_args": [ [ "k0=v0;k1=v1;k2=v2" ] ], "origin": "plan", "screen_ratio": 1.009196622275625, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "many_semicolon_segments_without_equals", "why": "Head version strips *every* segment before checking for \"=\". Base version only strips segments that actually contain \"=\". A header with many segments that lack \"=\" (e.g. \"a;b;c;...\") forces the new code to do O(n) extra strip operations.", "size_meaning": "Number of semicolon‑separated segments that do NOT contain \"=\".", "make_source": "def make(n):\n # n segments like \"seg0;seg1;...\" without \"=\"\n parts = [f'seg{i}' for i in range(n)]\n header = ';'.join(parts)\n return [[header]]", "example_args": [], "origin": "plan", "screen_ratio": 1.0846397547306883, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "many_empty_segments", "why": "The new version strips every segment before checking for '=', while the old version only stripped when '=' was present. A header with many empty segments (e.g. repeated ';;') forces the new code to allocate and strip a string for each empty piece, adding O(n) work the old code skipped.", "size_meaning": "Number of empty segments (each just ';').", "make_source": "def make(n):\n # header consisting of n empty segments: \";;\" repeated n times\n header = \";\" * (n + 1) # leading ';' gives n empty pieces after split\n return [[header]]", "example_args": [], "origin": "plan", "screen_ratio": 1.0950635955029355, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "malformed cookie header – many semicolon‑separated parts that do NOT contain \"=\"", "why": "The original version checks `if \"=\" in cookie:` *before* calling `strip()`. The new version unconditionally executes `cookie = cookie.strip()` for every part, then tests for \"=\". For inputs with a large number of parts lacking an equals sign the new code does an extra `strip()` call per part, making it slower.", "size_meaning": "n = number of malformed parts", "make_source": "def make(n):\n # Build a header like \"part0;part1;...;part{n-1}\" – no \"=\" anywhere\n header = \";\".join(f\"part{i}\" for i in range(n))\n return [[header]]", "example_args": [ [ "part0;part1;part2;part3" ] ], "origin": "plan", "screen_ratio": 1.0843783653713397, "valid_fraction": 1.0, "error": null }, { "id": "R1", "kind": "suspected_slowdown", "family": "many malformed fragments without \"=\" (invalid but accepted)", "why": "The parser still iterates over every `;`‑separated fragment. The new explicit `strip()` runs on each fragment even though the `if \"=\" in cookie:` guard skips the split. With many such fragments the extra `strip()` work dominates, making the new version slower.", "size_meaning": "n = number of malformed fragments.", "make_source": "def make(n):\n fragments = [f\"badtoken{i}\" for i in range(n)]\n header = \";\".join(fragments)\n return [[header]]", "example_args": [], "origin": "archive", "screen_ratio": 1.0835937892941376, "valid_fraction": 1.0, "error": null }, { "id": "R2", "kind": "suspected_slowdown", "family": "cookies surrounded by whitespace", "why": "The new code executes `cookie = cookie.strip()` for every split fragment *before* the `=` test, while the old version only stripped when `=` was present (`cookie.strip().split(...)`). Headers that contain many fragments without `=` (e.g. stray spaces or malformed tokens) pay the `strip()` cost without any benefit, slowing the path that the change was supposed to accelerate.", "size_meaning": "n = number of whitespace‑only fragments plus one real cookie.", "make_source": "def make(n):\n # n‑1 fragments that are just spaces, then one valid cookie\n fragments = [\" \" for _ in range(n-1)] + [\"session=abc123\"]\n header = \";\".join(fragments)\n return [[header]]", "example_args": [], "origin": "archive", "screen_ratio": 1.0668392964562166, "valid_fraction": 1.0, "error": null }, { "id": "R3", "kind": "suspected_slowdown", "family": "single cookie", "why": "Even with a single well‑formed cookie the new version allocates an extra empty dict (`cookies = {}` after the early‑return) and performs an explicit `strip()` call. The overhead is constant but measurable; it shows the regression on the smallest non‑empty input.", "size_meaning": "n is ignored – always one cookie.", "make_source": "def make(n):\n return [[\"userid=42\"]]", "example_args": [], "origin": "archive", "screen_ratio": 1.005607546458243, "valid_fraction": 1.0, "error": null }, { "id": "R4", "kind": "suspected_slowdown", "family": "few cookies but very large values", "why": "`strip()` copies the string; on huge values the extra explicit `strip()` copies the whole value a second time (the original `strip().split()` also copies once). The double copy makes the new version slower proportionally to the total length of cookie values.", "size_meaning": "n = length of the cookie value (in characters).", "make_source": "def make(n):\n # one cookie with a value of length n\n header = f\"big={'x'*n}\"\n return [[header]]", "example_args": [], "origin": "archive", "screen_ratio": 1.0056049575978068, "valid_fraction": 1.0, "error": null }, { "id": "F5.1", "kind": "suspected_slowdown", "family": "many_trailing_semicolons", "why": "The new version strips every segment before checking for '=', while the old version only stripped when '=' was present. A header with many empty segments (e.g. repeated ';;') forces the new code to allocate and strip a string for each empty piece, adding O(n) work the old code skipped.", "size_meaning": "Number of empty segments (each just ';').", "make_source": "def make(n):\n # Header ending with n semicolons produces n+1 empty trailing splits\n header = \"\";\" * (n + 2)\" \"", "example_args": [], "origin": "expand:F5", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: syntax error: unterminated string literal (detected at line 4) (line 4)" }, { "id": "F5.2", "kind": "suspected_slowdown", "family": "many_leading_spaces_stripped", "why": "The new version strips every segment before checking for '=', while the old version only stripped when '=' was present. A header with many empty segments (e.g. repeated ';;') forces the new code to allocate and strip a string for each empty piece, adding O(n) work the old code skipped.", "size_meaning": "Number of empty segments (each just ';').", "make_source": "def make(n):\n # Many segments starting with spaces, then null value (no =)\n header = ', '.join([' ;'] * n)", "example_args": [], "origin": "expand:F5", "screen_ratio": null, "valid_fraction": null, "error": "make(2): TypeError: 'NoneType' object is not iterable" }, { "id": "F4.1", "kind": "suspected_slowdown", "family": "many_semicolon_segments_with_plus_in_middle", "why": "Head version strips *every* segment before checking for \"=\". Base version only strips segments that actually contain \"=\". A header with many segments that lack \"=\" (e.g. \"a;b;c;...\") forces the new code to do O(n) extra strip operations.", "size_meaning": "Number of semicolon‑separated segments that do NOT contain \"=\".", "make_source": "def make(n):\n parts = []\n for i in range(n):\n s = f'seg{i}+{i+5}'\n parts.append(s)\n header = ';'.join(parts)\n return [[header]]", "example_args": [], "origin": "expand:F4", "screen_ratio": 1.0842533733254556, "valid_fraction": 1.0, "error": null }, { "id": "F4.2", "kind": "suspected_slowdown", "family": "many_colon_sep_keyval_no_value", "why": "Head version strips *every* segment before checking for \"=\". Base version only strips segments that actually contain \"=\". A header with many segments that lack \"=\" (e.g. \"a;b;c;...\") forces the new code to do O(n) extra strip operations.", "size_meaning": "Number of semicolon‑separated segments that do NOT contain \"=\".", "make_source": "def make(n):\n key_val_pairs = [f'key{i}:' for i in range(n)]\n header = ','.join(key_val_pairs)\n return [[header]]", "example_args": [], "origin": "expand:F4", "screen_ratio": 1.0121220505817177, "valid_fraction": 1.0, "error": null }, { "id": "F4.3", "kind": "suspected_slowdown", "family": "repeated_empty_segments_at_start_and_end", "why": "Head version strips *every* segment before checking for \"=\". Base version only strips segments that actually contain \"=\". A header with many segments that lack \"=\" (e.g. \"a;b;c;...\") forces the new code to do O(n) extra strip operations.", "size_meaning": "Number of semicolon‑separated segments that do NOT contain \"=\".", "make_source": "def make(n):\n empty_count = max(0, n // 2)\n normal_segments = ['normal'] * ((n + 1) // 2)\n all_parts = ([''] * empty_count) + normal_segments + ([''] * empty_count)\n header = ';'.join(all_parts)\n return [[header]]}]}, {", "example_args": [], "origin": "expand:F4", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: syntax error: unmatched '}' (line 7)" } ], "findings": [ { "family_id": "F5", "kind": "suspected_slowdown", "family": "many_empty_segments", "why": "The new version strips every segment before checking for '=', while the old version only stripped when '=' was present. A header with many empty segments (e.g. repeated ';;') forces the new code to allocate and strip a string for each empty piece, adding O(n) work the old code skipped.", "origin": "plan", "screen_ratio": 1.0950635955029355, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 42038.315, "head_ir": 43141.425, "input_chars": 7.0, "valid_fraction": 1.0, "ratio": 1.0262405855229926, "judged": true }, { "n": 4, "base_ir": 42635.3025, "head_ir": 44187.425, "input_chars": 9.0, "valid_fraction": 1.0, "ratio": 1.0364046320534492, "judged": true }, { "n": 16, "base_ir": 46703.5775, "head_ir": 51143.3875, "input_chars": 21.0, "valid_fraction": 1.0, "ratio": 1.0950635955029355, "judged": true }, { "n": 64, "base_ir": 64529.64, "head_ir": 80279.45, "input_chars": 69.0, "valid_fraction": 1.0, "ratio": 1.2440709416633968, "judged": true }, { "n": 256, "base_ir": 133564.81578947368, "head_ir": 194590.31578947368, "input_chars": 261.0, "valid_fraction": 1.0, "ratio": 1.456898020929322, "judged": true }, { "n": 1024, "base_ir": 398991.34210526315, "head_ir": 640353.7894736842, "input_chars": 1029.0, "valid_fraction": 1.0, "ratio": 1.604931540857205, "judged": true }, { "n": 4096, "base_ir": 1591178.375, "head_ir": 2551484.125, "input_chars": 4101.0, "valid_fraction": 1.0, "ratio": 1.6035186029976054, "judged": true } ], "slope_base": 0.5475252965420538, "slope_head": 0.6272788678221376, "top_inputs": [ { "args": [ ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;" ], "cpu_ratio": 1.339, "base_ir": 100577.46610169491, "head_ir": 139513.14406779662, "ir_ratio": 1.387121285465011, "outcome": "ok" }, { "args": [ ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;" ], "cpu_ratio": 1.337, "base_ir": 109769.48039215687, "head_ir": 154815.48039215687, "ir_ratio": 1.410369073799666, "outcome": "ok" } ], "error": null, "verdict": "slower", "worst_ratio": 1.604931540857205, "worst_search": { "args": [ ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;" ], "cpu_ratio": 1.337, "base_ir": 109769.48039215687, "head_ir": 154815.48039215687, "ir_ratio": 1.410369073799666, "outcome": "ok" } }, { "family_id": "F4", "kind": "suspected_slowdown", "family": "many_semicolon_segments_without_equals", "why": "Head version strips *every* segment before checking for \"=\". Base version only strips segments that actually contain \"=\". A header with many segments that lack \"=\" (e.g. \"a;b;c;...\") forces the new code to do O(n) extra strip operations.", "origin": "plan", "screen_ratio": 1.0846397547306883, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 41939.5775, "head_ir": 42649.3875, "input_chars": 13.0, "valid_fraction": 1.0, "ratio": 1.016924586328987, "judged": true }, { "n": 4, "base_ir": 43052.5775, "head_ir": 44262.3875, "input_chars": 23.0, "valid_fraction": 1.0, "ratio": 1.0281007565691043, "judged": true }, { "n": 16, "base_ir": 50269.64, "head_ir": 54524.45, "input_chars": 89.0, "valid_fraction": 1.0, "ratio": 1.0846397547306883, "judged": true }, { "n": 64, "base_ir": 81127.91346153847, "head_ir": 97382.18269230769, "input_chars": 377.0, "valid_fraction": 1.0, "ratio": 1.2003535964039693, "judged": true }, { "n": 256, "base_ir": 205038.0909090909, "head_ir": 270150.7272727273, "input_chars": 1685.0, "valid_fraction": 1.0, "ratio": 1.3175636101318646, "judged": true }, { "n": 1024, "base_ir": 689365.0, "head_ir": 945682.75, "input_chars": 7085.0, "valid_fraction": 1.0, "ratio": 1.3718171795783076, "judged": true }, { "n": 4096, "base_ir": 2765024.25, "head_ir": 3795769.25, "input_chars": 31661.0, "valid_fraction": 1.0, "ratio": 1.3727797324019853, "judged": true } ], "slope_base": 0.5266393361206952, "slope_head": 0.5708123012049002, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.3727797324019853, "worst_search": null }, { "family_id": "F6", "kind": "suspected_slowdown", "family": "malformed cookie header – many semicolon‑separated parts that do NOT contain \"=\"", "why": "The original version checks `if \"=\" in cookie:` *before* calling `strip()`. The new version unconditionally executes `cookie = cookie.strip()` for every part, then tests for \"=\". For inputs with a large number of parts lacking an equals sign the new code does an extra `strip()` call per part, making it slower.", "origin": "plan", "screen_ratio": 1.0843783653713397, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 41960.5775, "head_ir": 42670.3875, "input_chars": 15.0, "valid_fraction": 1.0, "ratio": 1.01691611608539, "judged": true }, { "n": 4, "base_ir": 43092.5775, "head_ir": 44302.3875, "input_chars": 27.0, "valid_fraction": 1.0, "ratio": 1.028074672488551, "judged": true }, { "n": 16, "base_ir": 50427.61827956989, "head_ir": 54682.61827956989, "input_chars": 105.0, "valid_fraction": 1.0, "ratio": 1.0843783653713397, "judged": true }, { "n": 64, "base_ir": 81903.98888888888, "head_ir": 98144.14444444445, "input_chars": 441.0, "valid_fraction": 1.0, "ratio": 1.1982828403826216, "judged": true }, { "n": 256, "base_ir": 207478.1, "head_ir": 272131.65, "input_chars": 1941.0, "valid_fraction": 1.0, "ratio": 1.3116162621500775, "judged": true }, { "n": 1024, "base_ir": 700383.25, "head_ir": 956888.25, "input_chars": 8109.0, "valid_fraction": 1.0, "ratio": 1.3662352005134333, "judged": true }, { "n": 4096, "base_ir": 2793083.75, "head_ir": 3803968.25, "input_chars": 35757.0, "valid_fraction": 1.0, "ratio": 1.3619241635701043, "judged": true } ], "slope_base": 0.5300648887785131, "slope_head": 0.5734420109497845, "top_inputs": [ { "args": [ [ "j7yzsofc2g58" ] ], "cpu_ratio": 1.107, "base_ir": 46430.9225, "head_ir": 46635.7325, "ir_ratio": 1.0044110689379475, "outcome": "exc:builtins.AttributeError" }, { "args": [ [ "mg0528ui8o5casqq24av" ] ], "cpu_ratio": 1.033, "base_ir": 46430.9225, "head_ir": 46635.7325, "ir_ratio": 1.0044110689379475, "outcome": "exc:builtins.AttributeError" } ], "error": null, "verdict": "slower", "worst_ratio": 1.3662352005134333, "worst_search": { "args": [ [ "j7yzsofc2g58" ] ], "cpu_ratio": 1.107, "base_ir": 46430.9225, "head_ir": 46635.7325, "ir_ratio": 1.0044110689379475, "outcome": "exc:builtins.AttributeError" } }, { "family_id": "F4.1", "kind": "suspected_slowdown", "family": "many_semicolon_segments_with_plus_in_middle", "why": "Head version strips *every* segment before checking for \"=\". Base version only strips segments that actually contain \"=\". A header with many segments that lack \"=\" (e.g. \"a;b;c;...\") forces the new code to do O(n) extra strip operations.", "origin": "expand:F4", "screen_ratio": 1.0842533733254556, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 41982.5775, "head_ir": 42692.3875, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 1.016907251585494, "judged": true }, { "n": 4, "base_ir": 43151.64, "head_ir": 44346.45, "input_chars": 31.0, "valid_fraction": 1.0, "ratio": 1.0276886347772645, "judged": true }, { "n": 16, "base_ir": 50674.409395973154, "head_ir": 54943.89932885906, "input_chars": 132.0, "valid_fraction": 1.0, "ratio": 1.0842533733254556, "judged": true }, { "n": 64, "base_ir": 83074.67142857143, "head_ir": 99354.67142857143, "input_chars": 564.0, "valid_fraction": 1.0, "ratio": 1.1959682743253186, "judged": true }, { "n": 256, "base_ir": 214406.35714285713, "head_ir": 278314.78571428574, "input_chars": 2609.0, "valid_fraction": 1.0, "ratio": 1.2980715190680983, "judged": true }, { "n": 1024, "base_ir": 734188.0, "head_ir": 989381.25, "input_chars": 11110.0, "valid_fraction": 1.0, "ratio": 1.3475857001204052, "judged": true }, { "n": 4096, "base_ir": 2972129.25, "head_ir": 3992042.25, "input_chars": 51046.0, "valid_fraction": 1.0, "ratio": 1.343159033208263, "judged": true } ], "slope_base": 0.5221943756459514, "slope_head": 0.5622911004648111, "top_inputs": [ { "args": [ "򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;4K򭬲I𽼛QÑ]»ìeE|\r¸񋠲¯;•Û.񱟤;‡K\u0007RØQ򮌦䤕ˆ€;󨓹\u001d𰡦\u0000N\u0010v°񧅇\u001d𻺽p;@¦Ã;ú;򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;4K򭬲I𽼛QÑ]»ìeE|\r¸񋠲¯;•Û.񱟤" ], "cpu_ratio": 1.218, "base_ir": 79653.1, "head_ir": 95151.3, "ir_ratio": 1.1945712094067902, "outcome": "ok" }, { "args": [ "򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;4K򭬲I𽼛QÑ]»ìeE|\r¸񋠲¯;•Û.񱟤;‡K\u0007RØQ򮌦䤕ˆ€;󨓹\u001d𰡦\u0000N\u0010v°񧅇\u001d𻺽p;@¦Ã;ú;򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;4K򭬲I𽼛QÑ]»ìeE|\r¸񋠲¯;•Û.񱟤;‡K\u0007RØQ򮌦䤕ˆ€;󨓹\u001d𰡦\u0000N\u0010v°񧅇\u001d𻺽p;@¦Ã;ú;𥤆«éñš\u000e\r򂣏\u0003Wæ;񠌭𘣼򶵔µ;¯󀲯k›¸”𲓎;򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016" ], "cpu_ratio": 1.214, "base_ir": 79660.6, "head_ir": 94453.05, "ir_ratio": 1.1856934293741197, "outcome": "ok" } ], "error": null, "verdict": "slower", "worst_ratio": 1.3475857001204052, "worst_search": { "args": [ "򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;4K򭬲I𽼛QÑ]»ìeE|\r¸񋠲¯;•Û.񱟤;‡K\u0007RØQ򮌦䤕ˆ€;󨓹\u001d𰡦\u0000N\u0010v°񧅇\u001d𻺽p;@¦Ã;ú;򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;򏣉å…N\u0001O;Ù;U\rÛ;ð򠪉âä,š…󃍵›iÈ;å?;IˆRX;ÂL€­½®񌡒\u0016;IˆRX;Ýùfx`ƒ󃟔ÎÙê2¡;ÒÑç\u0013\u0007󙯝¼9;󶢝h򢄼\u0016C\u0007;Ùw􃰟déµ4\u00064\u0002k¾U򘦸;ù󝿊­ý;ôæ򉺥ʞêã«Ä;4K򭬲I𽼛QÑ]»ìeE|\r¸񋠲¯;•Û.񱟤" ], "cpu_ratio": 1.218, "base_ir": 79653.1, "head_ir": 95151.3, "ir_ratio": 1.1945712094067902, "outcome": "ok" } }, { "family_id": "F2", "kind": "claimed_speedup", "family": "many_well_formed_cookies", "why": "Typical cookie headers contain many key=value pairs. The change moves the strip outside the equality test, which the author likely expected to avoid repeated attribute look‑ups and make the hot path tighter.", "origin": "plan", "screen_ratio": 1.0072629307261658, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 46379.5775, "head_ir": 46643.3875, "input_chars": 16.0, "valid_fraction": 1.0, "ratio": 1.0056880638897583, "judged": true }, { "n": 4, "base_ir": 52216.64, "head_ir": 52511.45, "input_chars": 30.0, "valid_fraction": 1.0, "ratio": 1.0056459013831607, "judged": true }, { "n": 16, "base_ir": 88944.81089743589, "head_ir": 89590.81089743589, "input_chars": 126.0, "valid_fraction": 1.0, "ratio": 1.0072629307261658, "judged": true }, { "n": 64, "base_ir": 237371.65714285715, "head_ir": 239241.65714285715, "input_chars": 558.0, "valid_fraction": 1.0, "ratio": 1.0078779413789682, "judged": true }, { "n": 256, "base_ir": 833412.3571428572, "head_ir": 840490.7857142857, "input_chars": 2598.0, "valid_fraction": 1.0, "ratio": 1.0084933088773667, "judged": true }, { "n": 1024, "base_ir": 3194014.25, "head_ir": 3216655.75, "input_chars": 11094.0, "valid_fraction": 1.0, "ratio": 1.007088728549035, "judged": true }, { "n": 4096, "base_ir": 12874764.75, "head_ir": 12973831.25, "input_chars": 51030.0, "valid_fraction": 1.0, "ratio": 1.0076946260319048, "judged": true } ], "slope_base": 0.707566564849701, "slope_head": 0.7078163998816763, "top_inputs": [], "error": null, "verdict": "neutral", "worst_ratio": 1.0084933088773667, "worst_search": null } ], "seconds": { "plan": 13.09, "screen": 334.81, "full": 378.58, "total": 726.49 } } } } ] }, { "id": "caddy-feeds#4", "pr": 4, "title": "⚡️ Speed up function `is_excluded` by 17,376%", "base": "f5e3d0c28fe86f51150194ba903055ab6a146a69", "head": "d94d92d5d9dafa5b20b06ab7b7f129952309187e", "merged": "2025-02-09", "after_ultra_release": false, "known": [], "repo": "https://github.com/fabriziosalmi/caddy-feeds", "install": { "mode": "src", "subdir": "scripts", "deps": [ "requests", "pyyaml", "aiohttp", "prometheus-client" ], "exclude_newer": "2025-02-10" }, "targets": [ { "target": "ip:is_excluded", "run_id": "bt-caddy-feeds#4-081228", "image": "1bafd31e-3df4-4239-8ac7-13dd73b31ec0", "backend": "contree", "usd": 0.0369, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 12.3, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 7.9, "error": null }, "breakwater": { "classes": [ "P2|outcome|ok|exc:builtins.TypeError|nested" ], "known": [], "seconds": 24.0, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 7.5, "error": null } }, "findings": [ { "signature": "P2|outcome|ok|exc:builtins.TypeError|nested", "probe": "P2", "detail": "outcome", "args": [ [ [ ":" ] ] ], "base": { "status": "ok", "obs": { "value": true } }, "head": { "status": "exc", "exc": "builtins.TypeError" }, "found_by": [ "A1/s3", "batch" ] } ], "perf": { "slower_families": [], "worst_valid_ratio": null, "worst_family": null, "speedup_ratio_small": 0.003160293020624661, "report": { "run_id": "bt-caddy-feeds#4-081228", "target": "ip:is_excluded", "image": "1bafd31e-3df4-4239-8ac7-13dd73b31ec0", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "repeated_same_ipv4", "why": "The same public IPv4 address is checked many times; the added LRU cache turns the first miss into a hit for all subsequent calls, so the head version should be faster.", "size_meaning": "Number of repeated checks (n) – the generator returns up to 8 identical calls, cost grows with n because the caller will repeat the list n times.", "make_source": "def make(n):\n ip = \"20.0.0.1\" # public, not in EXCLUDE_RANGES\n # return up to 8 identical argument lists; the harness can repeat the list n times\n return [[ip] for _ in range(min(n, 8))]", "example_args": [], "origin": "plan", "screen_ratio": 0.003161057396159, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "Repeated identical IP address – the cache should turn the second and later calls into O(1) look‑ups.", "why": "The added @lru_cache stores the result after the first evaluation. Subsequent calls with the same argument hit the cache and avoid the ipaddress parsing + overlap loop entirely.", "size_meaning": "Number of calls (n) – the first call does the full work, the remaining n‑1 calls are cache hits.", "make_source": "def make(n):\n ip = '8.8.8.8' # public, not excluded\n return [[ip] for _ in range(max(1, min(n, 8)))]", "example_args": [ [ [ "8.8.8.8" ], [ "8.8.8.8" ], [ "8.8.8.8" ] ] ], "origin": "plan", "screen_ratio": 0.0031605623280493017, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "repeated_same_ip", "why": "The new version adds an LRU cache. When the same IP address is checked many times, the first call does the full overlap scan and subsequent calls are O(1) cache hits. The base version repeats the full scan every time.", "size_meaning": "Number of calls (n) – we emit up to 8 calls, each with the identical address.", "make_source": "def make(n):\n # repeat the same public, non‑excluded IPv4 address\n ip = \"8.8.8.8\"\n calls = min(n, 8)\n return [[ip] for _ in range(calls)]", "example_args": [], "origin": "plan", "screen_ratio": 0.0031605577215178746, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "unique_ipv4", "why": "Every call receives a distinct, valid public IPv4 address. The cache never hits, so each call pays the extra dictionary lookup, hashing, and LRU bookkeeping on top of the original overlap test.", "size_meaning": "Number of distinct IPs (n) – the generator returns up to 8 different IPs; the harness can repeat the list to reach n total calls.", "make_source": "def make(n):\n base = 20_000_001 # 20.0.0.1 as integer\n ips = []\n for i in range(min(n, 8)):\n # construct a.b.c.d from integer\n val = base + i\n a = (val >> 24) & 0xFF\n b = (val >> 16) & 0xFF\n c = (val >> 8) & 0xFF\n d = val & 0xFF\n ips.append([f\"{a}.{b}.{c}.{d}\"])\n return ips", "example_args": [], "origin": "plan", "screen_ratio": 0.0029839806425204893, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "Many distinct, non‑excluded IPv4 addresses – each call misses the cache and pays the cache‑lookup overhead plus the full ipaddress/overlap work.", "why": "The cache can hold at most 1024 entries, but with a stream of unique addresses every call is a miss. The wrapper adds a dict lookup and function‑call overhead that the original code did not have.", "size_meaning": "Number of distinct IPs (n, ≤8). Cost grows linearly with n because each call does a full check.", "make_source": "def make(n):\n base = 1_000_000 # 1.0.0.0/8 is excluded, start above it\n ips = []\n for i in itertools.islice(range(base, base + max(1, min(n, 8))), max(1, min(n, 8))):\n a = (i >> 24) & 0xFF\n b = (i >> 16) & 0xFF\n c = (i >> 8) & 0xFF\n d = i & 0xFF\n ips.append([f'{a}.{b}.{c}.{d}'])\n return ips", "example_args": [ [ [ "15.0.0.0" ], [ "15.0.0.1" ], [ "15.0.0.2" ] ] ], "origin": "plan", "screen_ratio": 0.017936193458917248, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "many_distinct_valid_ips", "why": "Each distinct, non‑excluded address causes a cache miss. The LRU wrapper adds a dictionary lookup, key creation and possible eviction logic on top of the original linear scan over EXCLUDE_NETWORKS. With no repeats the base version avoids that overhead.", "size_meaning": "Number of distinct IPs (n) – we emit up to 8 distinct public addresses.", "make_source": "def make(n):\n calls = min(n, 8)\n # Use a public / non‑excluded block (8.0.0.0/8 is not in EXCLUDE_RANGES)\n return [[f\"8.0.0.{i}\"] for i in range(calls)]", "example_args": [], "origin": "plan", "screen_ratio": 0.0030291575488036035, "valid_fraction": 1.0, "error": null } ], "findings": [ { "family_id": "F3", "kind": "claimed_speedup", "family": "repeated_same_ip", "why": "The new version adds an LRU cache. When the same IP address is checked many times, the first call does the full overlap scan and subsequent calls are O(1) cache hits. The base version repeats the full scan every time.", "origin": "plan", "screen_ratio": 0.0031605577215178746, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 298862.70875, "head_ir": 1171.84875, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.003921026998989884, "judged": true }, { "n": 4, "base_ir": 298706.6425, "head_ir": 1014.5225, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.0033963841296230967, "judged": true }, { "n": 16, "base_ir": 298606.771484375, "head_ir": 943.6848958333334, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.003160293020624661, "judged": true }, { "n": 64, "base_ir": 298606.7698567708, "head_ir": 943.7643229166666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.003160559030089475, "judged": true }, { "n": 256, "base_ir": 298606.6829427083, "head_ir": 943.685546875, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.003160296137967075, "judged": true }, { "n": 1024, "base_ir": 298606.6826171875, "head_ir": 943.6884765625, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.003160305952604231, "judged": true }, { "n": 4096, "base_ir": 298606.771484375, "head_ir": 943.763671875, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.003160556832598097, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "faster", "worst_ratio": 0.003921026998989884, "worst_search": null } ], "seconds": { "plan": 11.43, "screen": 300.54, "full": 224.83, "total": 536.81 } } } } ] }, { "id": "patterns#9", "pr": 9, "title": "⚡️ Speed up function `validate_regex` by 9,172%", "base": "1a98411808332cac22183dfc4182a74759d0cca4", "head": "1a4a2d4e42342721e2e46f479864e8741042ebf6", "merged": "2025-02-09", "after_ultra_release": false, "known": [], "repo": "https://github.com/fabriziosalmi/patterns", "install": { "mode": "src", "subdir": "", "deps": [], "exclude_newer": "2025-02-10" }, "targets": [ { "target": "json2haproxy:validate_regex", "run_id": "bt-patterns#9-728c64", "image": "bcc79688-d7fb-4ce5-921c-c5f853d78731", "backend": "contree", "usd": 0.0309, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 9.0, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 5.5, "error": null }, "breakwater": { "classes": [], "known": [], "seconds": 23.8, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 4.2, "error": null } }, "findings": [], "perf": { "slower_families": [], "worst_valid_ratio": null, "worst_family": null, "speedup_ratio_small": 0.2277454522428054, "report": { "run_id": "bt-patterns#9-728c64", "target": "json2haproxy:validate_regex", "image": "bcc79688-d7fb-4ce5-921c-c5f853d78731", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "repeated identical valid regex", "why": "The new @lru_cache stores the result of re.compile for each distinct pattern. When the same pattern is validated many times (common in rule sets that reuse signatures), the cache avoids recompilation and yields the intended speed‑up.", "size_meaning": "Number of repeated validations (capped at 8 because the generator may return at most 8 argument lists).", "make_source": "def make(n):\n pattern = r\"a.*b\"\n # Return up to 8 identical calls; cost grows with n (more repetitions).\n return [[pattern] for _ in range(min(n, 8))]", "example_args": [ [ "a.*b" ], [ "a.*b" ], [ "a.*b" ] ], "origin": "plan", "screen_ratio": 0.23065953278424117, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "Repeated identical regex patterns (duplicates). The cache returns the compiled‑result on the 2nd…Nth call, so total time grows only with the first compilation.", "why": "The added @lru_cache stores the boolean result for each distinct pattern string. When the same pattern appears many times (common in OWASP rule sets where a rule is referenced repeatedly), the head version avoids re‑compiling the regex on every call, while the base version recompiles each time.", "size_meaning": "n = length of the pattern string (number of characters). The generator produces up to 8 calls, all with the same pattern of length n, so total work scales with n.", "make_source": "def make(n):\n # a simple but non‑trivial pattern that compiles quickly\n pat = 'a' * n + '.*b'\n # return 8 identical calls – the cost grows with n because the pattern is longer\n return [[pat]] * 8", "example_args": [], "origin": "plan", "screen_ratio": 0.2277454522428054, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "validate_regex called many times with the identical pattern – the cache should turn the second and later compilations into cheap dictionary look‑ups.", "why": "The @lru_cache decorator stores the result of the first compilation; subsequent calls with the same string avoid re‑compiling the regex.", "size_meaning": "n = how many times the same pattern is validated (1‑8). Larger n gives more cache hits.", "make_source": "def make(n):\n # a simple but non‑trivial pattern that compiles successfully\n pat = r'(?i)foo(bar|baz){2,5}'\n k = min(max(1, n), 8)\n return [[pat] for _ in range(k)]", "example_args": [], "origin": "plan", "screen_ratio": 0.22833910485472247, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "many distinct valid regexes", "why": "Each distinct pattern incurs a cache miss, a dict lookup, and a re.compile call. With many unique patterns the cache grows, increasing memory pressure and hash‑table overhead without any reuse benefit.", "size_meaning": "Number of distinct patterns (capped at 8).", "make_source": "def make(n):\n # Produce up to 8 different simple valid regexes.\n return [[f\"a{i}b\"] for i in range(min(n, 8))]", "example_args": [ [ "a0b" ], [ "a1b" ], [ "a2b" ] ], "origin": "plan", "screen_ratio": 0.22615529869614667, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "Many distinct, large patterns. Each call hashes a long string and performs a dict lookup before the inevitable re.compile, adding constant‑time overhead that the base version does not have.", "why": "The cache must compute the hash of the pattern string and look it up in the LRU dictionary. For a pattern of length n this hashing is O(n). When every call receives a brand‑new pattern, the cache never hits, so the head version does the hash+lookup *plus* the regex compilation, whereas the base version only does the compilation.", "size_meaning": "n = length of each distinct pattern. The generator creates 8 unique patterns, each of length proportional to n, so total work grows with n.", "make_source": "def make(n):\n # produce 8 different long patterns, e.g. varying a repeated character\n base = 'x' * n\n patterns = [base[:i] + '.' * (n - i) + 'z' for i in range(8)]\n return [[p] for p in patterns]", "example_args": [], "origin": "plan", "screen_ratio": 0.22569346082947428, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "validate_regex receives a different pattern on every call – the cache is never hit, so the wrapper adds dictionary‑lookup and bookkeeping overhead on top of the compilation cost.", "why": "With an unlimited‑size LRU cache each distinct pattern is stored after the first miss. The extra function‑call wrapper and dict insertion make each call a little slower than the bare version.", "size_meaning": "n = number of distinct patterns validated (1‑8). More distinct patterns → more cache misses.", "make_source": "def make(n):\n k = min(max(1, n), 8)\n # generate short, distinct, valid regexes\n patterns = []\n for i in range(k):\n # e.g. 'a{1}', 'ab{2}', 'abc{3}', ...\n pat = ''.join(chr(ord('a') + j) for j in range(i + 1)) + '{' + str(i + 1) + '}'\n patterns.append([pat])\n return patterns", "example_args": [], "origin": "plan", "screen_ratio": 0.22664317195510447, "valid_fraction": 1.0, "error": null } ], "findings": [ { "family_id": "F2", "kind": "claimed_speedup", "family": "Repeated identical regex patterns (duplicates). The cache returns the compiled‑result on the 2nd…Nth call, so total time grows only with the first compilation.", "why": "The added @lru_cache stores the boolean result for each distinct pattern string. When the same pattern appears many times (common in OWASP rule sets where a rule is referenced repeatedly), the head version avoids re‑compiling the regex on every call, while the base version recompiles each time.", "origin": "plan", "screen_ratio": 0.2277454522428054, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 3953.8834375, "head_ir": 916.3834375, "input_chars": 9.0, "valid_fraction": 1.0, "ratio": 0.23176794460066832, "judged": true }, { "n": 4, "base_ir": 3953.2587890625, "head_ir": 915.7587890625, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.23164655741641155, "judged": true }, { "n": 16, "base_ir": 3957.891875, "head_ir": 901.391875, "input_chars": 23.0, "valid_fraction": 1.0, "ratio": 0.2277454522428054, "judged": true }, { "n": 64, "base_ir": 3942.455882352941, "head_ir": 904.9558823529412, "input_chars": 71.0, "valid_fraction": 1.0, "ratio": 0.2295411564156412, "judged": true }, { "n": 256, "base_ir": 3969.7986111111113, "head_ir": 932.2986111111111, "input_chars": 263.0, "valid_fraction": 1.0, "ratio": 0.23484783548003937, "judged": true }, { "n": 1024, "base_ir": 4058.75, "head_ir": 1021.25, "input_chars": 1031.0, "valid_fraction": 1.0, "ratio": 0.2516168771173391, "judged": true }, { "n": 4096, "base_ir": 4434.75, "head_ir": 1378.25, "input_chars": 4103.0, "valid_fraction": 1.0, "ratio": 0.3107841479226563, "judged": true } ], "slope_base": 0.01448696535713944, "slope_head": 0.053613716648372396, "top_inputs": [], "error": null, "verdict": "faster", "worst_ratio": 0.3107841479226563, "worst_search": null } ], "seconds": { "plan": 9.69, "screen": 89.25, "full": 74.69, "total": 173.64 } } } } ] }, { "id": "patterns#12", "pr": 12, "title": "⚡️ Speed up function `sanitize_pattern` by 11,547%", "base": "1a4a2d4e42342721e2e46f479864e8741042ebf6", "head": "fb284892f1cd90ee55fd99a8383f537fef1d520e", "merged": "2025-02-09", "after_ultra_release": false, "known": [], "repo": "https://github.com/fabriziosalmi/patterns", "install": { "mode": "src", "subdir": "", "deps": [], "exclude_newer": "2025-02-10" }, "targets": [ { "target": "json2nginx:sanitize_pattern", "run_id": "bt-patterns#12-0dc7f0", "image": "87c6903e-46fd-4cf5-a313-2b45b6b1248c", "backend": "contree", "usd": 0.051, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 9.6, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 6.1, "error": null }, "breakwater": { "classes": [ "P2|outcome|exc:builtins.AttributeError|exc:builtins.TypeError|nested" ], "known": [], "seconds": 23.6, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 4.5, "error": null } }, "findings": [ { "signature": "P2|outcome|exc:builtins.AttributeError|exc:builtins.TypeError|nested", "probe": "P2", "detail": "outcome", "args": [ [ [ [ "pattern", "0" ] ] ] ], "base": { "status": "exc", "exc": "builtins.AttributeError" }, "head": { "status": "exc", "exc": "builtins.TypeError" }, "found_by": [ "A1/s4", "A1/s3", "batch" ] } ], "perf": { "slower_families": [], "worst_valid_ratio": null, "worst_family": null, "speedup_ratio_small": 0.0575368320817423, "report": { "run_id": "bt-patterns#12-0dc7f0", "target": "json2nginx:sanitize_pattern", "image": "87c6903e-46fd-4cf5-a313-2b45b6b1248c", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "The new code adds LRU caches on both regex compilation and the whole sanitisation routine. When the same pattern string is seen many times the cache hits avoid ", "why": "The new code adds LRU caches on both regex compilation and the whole sanitisation routine. When the same pattern string is seen many times the cache hits avoid re‑compilation and re‑sanitising, which is the optimisation the author intended.", "size_meaning": "n = number of calls to sanitize_pattern (each call receives the identical pattern). Cost grows linearly with n because the function is invoked n times, but after the first call the cache eliminates the expensive work.", "make_source": "def make(n):\n # a single realistic OWASP pattern that passes all checks\n pat = r\"@rx \\\\d+\"\n # return n argument‑lists, each a one‑element list for the positional call\n return [[pat] for _ in range(n)]", "example_args": [ [ "@rx \\d+" ], [ "@rx \\d+" ], [ "@rx \\d+" ] ], "origin": "plan", "screen_ratio": 0.09461483475371836, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "Repeated identical pattern strings – cache hits after the first call.", "why": "The change adds lru_cache to _sanitize_pattern and _compile_pattern. When the same pattern appears many times, the first call pays the full sanitisation cost and subsequent calls are served from the cache, giving a net speed‑up.", "size_meaning": "Number of repetitions (n). Cost grows linearly with n, but the marginal cost per call drops after the first.", "make_source": "def make(n):\n # a simple pattern that passes all checks\n pat = \"simplePattern\"\n return [[pat] for _ in range(n)]", "example_args": [ [ "simplePattern" ], [ "simplePattern" ], [ "simplePattern" ] ], "origin": "plan", "screen_ratio": 0.05824795356595517, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "Repeated identical pattern (cache hit)", "why": "The new version caches the result of _sanitize_pattern with lru_cache. When the same pattern is processed many times the cached result is returned instantly, while the old version re‑executes the whole validation and escaping logic each call.", "size_meaning": "Number of repetitions (n) – the benchmark will call the target n times with the same argument.", "make_source": "def make(n):\n # Return a list with a single argument list; the benchmark will invoke the target n times.\n pattern = \"foo\"\n return [[pattern]] * min(n, 8)", "example_args": [], "origin": "plan", "screen_ratio": 0.11111301545452745, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "When every input pattern is distinct the LRU caches never hit. Each call now pays the overhead of two cache look‑ups (one in `_sanitize_pattern`, one in `_compi", "why": "When every input pattern is distinct the LRU caches never hit. Each call now pays the overhead of two cache look‑ups (one in `_sanitize_pattern`, one in `_compile_pattern`) plus the function‑call wrapper, which is slower than the original straight‑line code that compiled the regex once and returned.", "size_meaning": "n = number of distinct patterns supplied. Cost grows with n because each pattern incurs a cache miss and the extra Python‑level indirection.", "make_source": "def make(n):\n # generate n different harmless patterns that are valid regexes\n return [[f\"literal_pat_{i}\"] for i in range(n)]", "example_args": [ [ "literal_pat_0" ], [ "literal_pat_1" ], [ "literal_pat_2" ] ], "origin": "plan", "screen_ratio": 0.061962936557114115, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "Many distinct pattern strings – each call misses the cache.", "why": "With a cold cache (or >128 distinct patterns) every call incurs the cache‑lookup overhead (hashing the key, dict lookup) plus the function‑call wrapper before doing the real work. The original code performed the work directly with no extra indirection.", "size_meaning": "Number of distinct patterns (n). Cost grows with n because each call is a cache miss.", "make_source": "def make(n):\n # generate n different patterns that are all valid regexes\n return [[f\"pat{i}\"] for i in range(n)]", "example_args": [ [ "pat0" ], [ "pat1" ], [ "pat2" ] ], "origin": "plan", "screen_ratio": 0.08740356073978935, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "Many distinct patterns exceeding cache capacity", "why": "Each unique pattern incurs a cache miss, insertion, and possible LRU eviction once the 128‑entry cache fills. The old version simply processes each pattern once with no dictionary overhead. For workloads with >128 different patterns the extra bookkeeping makes the new version slower.", "size_meaning": "Number of distinct patterns (n) – each pattern is different, causing a miss.", "make_source": "def make(n):\n # Produce up to 8 distinct patterns of growing length to keep work proportional to n.\n # Each pattern is unique, so the cache cannot reuse results.\n patterns = [f\"pattern_{i}_\" + \"x\" * (i % 5 + 1) for i in range(min(n, 8))]\n return [[p] for p in patterns]", "example_args": [], "origin": "plan", "screen_ratio": 0.06449275297591466, "valid_fraction": 1.0, "error": null } ], "findings": [ { "family_id": "F2", "kind": "claimed_speedup", "family": "Repeated identical pattern strings – cache hits after the first call.", "why": "The change adds lru_cache to _sanitize_pattern and _compile_pattern. When the same pattern appears many times, the first call pays the full sanitisation cost and subsequent calls are served from the cache, giving a net speed‑up.", "origin": "plan", "screen_ratio": 0.05824795356595517, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 28146.22, "head_ir": 1920.185, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 0.068221771875584, "judged": true }, { "n": 4, "base_ir": 28010.48, "head_ir": 1761.43375, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 0.06288481132776018, "judged": true }, { "n": 16, "base_ir": 27891.50480769231, "head_ir": 1624.623076923077, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 0.05824795356595517, "judged": true }, { "n": 64, "base_ir": 27837.484375, "head_ir": 1601.6806640625, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 0.0575368320817423, "judged": true }, { "n": 256, "base_ir": 27837.484375, "head_ir": 1601.6806640625, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 0.0575368320817423, "judged": true }, { "n": 1024, "base_ir": 27837.484375, "head_ir": 1601.6806640625, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 0.0575368320817423, "judged": true }, { "n": 4096, "base_ir": 27837.484375, "head_ir": 1601.6806640625, "input_chars": 17.0, "valid_fraction": 1.0, "ratio": 0.0575368320817423, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "faster", "worst_ratio": 0.068221771875584, "worst_search": null } ], "seconds": { "plan": 11.55, "screen": 85.6, "full": 70.91, "total": 168.06 } } } } ] }, { "id": "rendercv#17", "pr": 17, "title": "⚡️ Speed up function `validate_typst_dimension` by 59%", "base": "ddd75cabd6753939b4c3b18f40c061dae77c8021", "head": "7f7abe4ebf939e25ebac92b9d1d43f226e80b373", "merged": "2026-08-10", "after_ultra_release": true, "known": [], "repo": "https://github.com/IanHollow/rendercv", "install": { "exclude_newer": "2026-08-11" }, "targets": [ { "target": "rendercv.themes.options:validate_typst_dimension", "run_id": "bt-rendercv#17-945eef", "image": "aa55391a-3699-4da1-90cb-9549f152a03d", "backend": "contree", "usd": 0.0316, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 13.3, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 9.9, "error": null }, "breakwater": { "classes": [], "known": [], "seconds": 6.6, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 0.0, "error": null } }, "findings": [], "perf": { "slower_families": [], "worst_valid_ratio": null, "worst_family": null, "speedup_ratio_small": 0.5600697437602842, "report": { "run_id": "bt-rendercv#17-945eef", "target": "rendercv.themes.options:validate_typst_dimension", "image": "aa55391a-3699-4da1-90cb-9549f152a03d", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "valid dimensions with increasing numeric length", "why": "The change hoists the regex compilation out of the hot path; each call now reuses a pre‑compiled pattern, so the cost grows only with the match itself. Long valid strings exercise the matching loop and show the speed‑up.", "size_meaning": "n = number of decimal digits before the unit (the string length grows ≈ n).", "make_source": "def make(n):\n # produce n+1 test cases, each a valid dimension with 1..n+1 digits\n out = []\n for i in range(1, n + 2):\n # e.g. '12345cm'\n s = '1' + '0' * (i - 1) + 'cm'\n out.append([s])\n return out", "example_args": [ [ "1cm" ], [ "10cm" ], [ "100cm" ] ], "origin": "plan", "screen_ratio": 0.5875239215825396, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "Repeated calls with typical valid Typst dimensions (e.g. \"10cm\", \"12.5pt\", \"3em\")", "why": "The head version reuses a module‑level compiled regex (_DIMENSION_PATTERN), so each call only executes the fast fullmatch on the pre‑compiled pattern. The base version calls re.fullmatch with a literal pattern string each time; although the re module caches recent patterns, the first few calls still pay the compilation cost, making the head faster for a hot loop of many valid dimensions.", "size_meaning": "n = number of dimensions validated in the loop", "make_source": "\ndef make(n):\n # produce n distinct valid dimension strings\n units = [\"cm\", \"in\", \"pt\", \"mm\", \"ex\", \"em\"]\n vals = []\n for i, u in zip(range(n), itertools.cycle(units)):\n # alternate integer and decimal forms\n if i % 2 == 0:\n vals.append(f\"{i+1}{u}\")\n else:\n vals.append(f\"{i+1}.5{u}\")\n # each element is a single‑argument call list for validate_typst_dimension\n return [[v] for v in vals]", "example_args": [ [ "10cm" ], [ "12.5pt" ], [ "3em" ] ], "origin": "plan", "screen_ratio": 0.5694788009675633, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "valid typst dimensions of increasing numeric length", "why": "The head version reuses a pre‑compiled regex (_DIMENSION_PATTERN) while the base version calls re.fullmatch with a literal pattern each time. For repeated valid inputs the head avoids repeated pattern lookup/compilation, so it becomes faster as the number of calls grows.", "size_meaning": "n = number of digits in the numeric part of the dimension string", "make_source": "def make(n):\n # generate 1‑8 dimension strings whose numeric part has n digits\n # each string is a valid dimension, e.g. '12345cm'\n res = []\n for i in range(1, 9):\n num = '1' * n + str(i) # n+1 digits, still O(n)\n res.append([f'{num}cm'])\n return res", "example_args": [ [ "111cm" ], [ "1111cm" ], [ "11111cm" ] ], "origin": "plan", "screen_ratio": 0.6138530398627762, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "invalid strings that force a full scan before failing", "why": "Both versions must scan the entire input before rejecting it; the head version adds a trivial attribute lookup (_DIMENSION_PATTERN) but otherwise does the same work. For extremely long non‑matching strings the overhead of the extra attribute access can make the new code marginally slower. Marked as invalid because such inputs are not accepted by the validator.", "size_meaning": "n = length of the garbage prefix before the unit suffix.", "make_source": "def make(n):\n out = []\n for i in range(1, n + 2):\n # a string of 'x' repeated i times followed by a valid unit – never matches\n s = 'x' * i + 'cm'\n out.append([s])\n return out", "example_args": [ [ "xcm" ], [ "xxcm" ], [ "xxxcm" ] ], "origin": "plan", "screen_ratio": 0.6656152188650563, "valid_fraction": 0.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "A single call with a very long string that does *not* match the dimension pattern (e.g. \"9\" * n + \"px\")", "why": "Both versions must scan the whole input before rejecting it, but the head version adds an extra attribute lookup (_DIMENSION_PATTERN) and a bound‑method call (fullmatch) on the compiled object, while the base version calls the module‑level re.fullmatch which is a plain function. For extremely long inputs the constant‑factor overhead of the attribute access can make the head marginally slower.", "size_meaning": "n = length of the digit run before the non‑matching suffix", "make_source": "def make(n):\n # one argument list containing a long non‑matching string\n s = \"9\" * n + \"px\" # \"px\" is not an allowed unit\n return [[s]]", "example_args": [ [ "9999999999px" ] ], "origin": "plan", "screen_ratio": 0.941588103824428, "valid_fraction": 0.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "very long strings that do not match the dimension pattern (invalid)", "why": "Both versions must run the regex to discover the mismatch. The head version adds an extra attribute access (_DIMENSION_PATTERN.fullmatch) and a function call, while the base version calls re.fullmatch directly. For extremely long non‑matching inputs the constant‑factor overhead of the extra attribute lookup can make the head marginally slower.", "size_meaning": "n = length of the input string (number of characters before the first non‑digit)", "make_source": "def make(n):\n # produce 1‑8 long invalid strings: n 'a' characters followed by 'cm'\n # they are not valid dimensions because they lack leading digits\n res = []\n for i in range(1, 9):\n s = 'a' * n + 'cm'\n res.append([s])\n return res", "example_args": [ [ "aaaaaaaaacm" ], [ "aaaaaaaaaaaaacm" ], [ "aaaaaaaaaaaaaaaaacm" ] ], "origin": "plan", "screen_ratio": 0.6673849806035448, "valid_fraction": 0.0, "error": null } ], "findings": [ { "family_id": "F2", "kind": "claimed_speedup", "family": "Repeated calls with typical valid Typst dimensions (e.g. \"10cm\", \"12.5pt\", \"3em\")", "why": "The head version reuses a module‑level compiled regex (_DIMENSION_PATTERN), so each call only executes the fast fullmatch on the pre‑compiled pattern. The base version calls re.fullmatch with a literal pattern string each time; although the re module caches recent patterns, the first few calls still pay the compilation cost, making the head faster for a hot loop of many valid dimensions.", "origin": "plan", "screen_ratio": 0.5694788009675633, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": null, "head_ir": 3933.79875, "input_chars": 8.0, "valid_fraction": 1.0, "ratio": null, "judged": false }, { "n": 4, "base_ir": 6750.783125, "head_ir": 3780.909375, "input_chars": 8.0, "valid_fraction": 1.0, "ratio": 0.5600697437602842, "judged": true }, { "n": 16, "base_ir": 6751.865546218487, "head_ir": 3845.0417542016808, "input_chars": 8.4375, "valid_fraction": 1.0, "ratio": 0.5694784245748452, "judged": true }, { "n": 64, "base_ir": 6774.465680803572, "head_ir": 3850.6774553571427, "input_chars": 8.859375, "valid_fraction": 1.0, "ratio": 0.5684105045020147, "judged": true }, { "n": 256, "base_ir": 6774.610491071428, "head_ir": 3850.6495535714284, "input_chars": 8.859375, "valid_fraction": 1.0, "ratio": 0.5683942358968648, "judged": true }, { "n": 1024, "base_ir": 6774.77734375, "head_ir": 3850.5764508928573, "input_chars": 8.859375, "valid_fraction": 1.0, "ratio": 0.5683694467752755, "judged": true }, { "n": 4096, "base_ir": 6774.481863839285, "head_ir": 3850.7039620535716, "input_chars": 8.859375, "valid_fraction": 1.0, "ratio": 0.5684130593968808, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "faster", "worst_ratio": 0.5694784245748452, "worst_search": null } ], "seconds": { "plan": 8.64, "screen": 322.54, "full": 245.23, "total": 576.41 } } } } ] }, { "id": "rendercv#18", "pr": 18, "title": "⚡️ Speed up function `validate_date_field` by 47%", "base": "9ec5e7bd90913bce5b157e34b11a6e797fe57164", "head": "d5f419015d7370b104f2e7620819df1ec0ff64ee", "merged": "2026-08-10", "after_ultra_release": true, "known": [], "repo": "https://github.com/IanHollow/rendercv", "install": { "exclude_newer": "2026-08-11" }, "targets": [ { "target": "rendercv.data.models.computers:get_date_object", "run_id": "bt-rendercv#18-9197ee", "image": "54410864-aefc-4bb1-bbeb-5b5c15c9eda6", "backend": "contree", "usd": 0.0622, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 13.4, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 9.8, "error": null }, "breakwater": { "classes": [ "P2|outcome|exc:builtins.TypeError|exc:builtins.ValueError|nested" ], "known": [], "seconds": 49.8, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 7.6, "error": null } }, "findings": [ { "signature": "P2|outcome|exc:builtins.TypeError|exc:builtins.ValueError|nested", "probe": "P2", "detail": "outcome", "args": [ [ [ "!!!!" ] ] ], "base": { "status": "exc", "exc": "builtins.TypeError" }, "head": { "status": "exc", "exc": "builtins.ValueError" }, "found_by": [ "A1/s2", "batch" ] } ], "perf": { "slower_families": [ { "id": "F6", "family": "The literal string \"present\"", "verdict": "slower", "ladder": [ 1.104327506134762, 1.105942425781514, 1.1108399529368673, 1.1072494580227126, 1.1072290345741593, 1.1072326692748182, 1.1072253381090982 ], "origin": "plan" }, { "id": "R1", "family": "present_string", "verdict": "slower", "ladder": [ 1.104421315125742, 1.1059736266928542, 1.1108652617335306, 1.1072605156683655, 1.1072856408346339, 1.1072401786405646, 1.107228833732626 ], "origin": "archive" }, { "id": "F5", "family": "present-string", "verdict": "slower", "ladder": [ 1.1044004825607783, 1.105945986490329, 1.1108008733275014, 1.1072326259298082, 1.1072217723380606, 1.1072474141036113, 1.107249062828535 ], "origin": "plan" }, { "id": "F4", "family": "present_literal", "verdict": "slower", "ladder": [ 1.104441735280664, 1.105953159371775, 1.1108723236337363, 1.1072444303641191, 1.1072623530942078, 1.1072200320019574, 1.107253803899394 ], "origin": "plan" } ], "worst_valid_ratio": 1.1108723236337363, "worst_family": "present_literal", "speedup_ratio_small": 0.6407489010615026, "report": { "run_id": "bt-rendercv#18-9197ee", "target": "rendercv.data.models.computers:get_date_object", "image": "54410864-aefc-4bb1-bbeb-5b5c15c9eda6", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "valid_iso_dates", "why": "The rewrite removes compiled regular‑expression objects and replaces them with cheap length / character checks. For well‑formed ISO dates (YYYY‑MM‑DD, YYYY‑MM, YYYY) the new code avoids the regex engine entirely, so the change is intended to accelerate this family.", "size_meaning": "n = number of distinct valid dates", "make_source": "def make(n):\n # produce n distinct valid YYYY‑MM‑DD dates, e.g. 2000‑01‑01, 2001‑01‑01 …\n return [[f\"{2000+i:04d}-01-01\"] for i in range(n)]", "example_args": [ [ "2000-01-01" ], [ "2001-01-01" ] ], "origin": "plan", "screen_ratio": 0.6424007259085833, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "valid-full-date", "why": "The change removes regex matching for the common YYYY‑MM‑DD path; the old version runs three compiled‑regex fullmatches before hitting the YYYY‑MM‑DD branch, while the new version uses cheap length / index checks.", "size_meaning": "n different full‑date strings (year varies, month/day fixed)", "make_source": "\ndef make(n):\n # generate n distinct valid YYYY-MM-DD strings, e.g. 2000-01-01, 2001-01-01, …\n years = (f\"{2000+i:04d}-01-01\" for i in range(n))\n return [[y] for y in itertools.islice(years, n)]", "example_args": [], "origin": "plan", "screen_ratio": 0.6423101166340122, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "ISO full dates (YYYY-MM-DD)", "why": "The new code uses simple length and character checks instead of three pre‑compiled regex fullmatches. For well‑formed ISO dates the fast path is a single `len` test and two index checks, which is cheaper than the regex engine.", "size_meaning": "Number of dates supplied; each call processes one date.", "make_source": "def make(n):\n # generate n distinct valid YYYY‑MM‑DD dates\n base = 2000\n dates = []\n for i in range(n):\n year = base + (i % 100)\n month = (i % 12) + 1\n day = (i % 28) + 1\n dates.append([f\"{year:04d}-{month:02d}-{day:02d}\"])\n return dates", "example_args": [], "origin": "plan", "screen_ratio": 0.6422891505593947, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "present_literal", "why": "In the original version the literal string \"present\" was tested *before* any regex match, so it returned immediately. The new version first checks the length‑10, length‑7 and length‑4 patterns, then finally tests `date == \"present\"`. Each call therefore performs three cheap length/index checks that the old code skipped.", "size_meaning": "n = number of repetitions of the literal \"present\"", "make_source": "def make(n):\n return [[\"present\"] for _ in range(n)]", "example_args": [ [ "present" ], [ "present" ] ], "origin": "plan", "screen_ratio": 1.1108182414181655, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "present-string", "why": "In the new code the literal \"present\" is tested *after* the three length‑/index‑checks (len==10, len==7, len==4). The old version checked `date == \"present\"` immediately after the integer case, so the new version does extra Python work for every \"present\" input.", "size_meaning": "n repetitions of the literal \"present\" (cost grows linearly with n calls)", "make_source": "\ndef make(n):\n return [[\"present\"] for _ in range(n)]", "example_args": [], "origin": "plan", "screen_ratio": 1.110830555322604, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "The literal string \"present\"", "why": "In the original version the `date == \"present\"` test was performed *before* any regex work, so it returned immediately. The new version first evaluates three `len`/index checks and a full `str.isdigit()` scan (7 characters) before reaching the equality test, adding overhead for this valid input.", "size_meaning": "Number of repetitions; each call processes the same literal.", "make_source": "def make(n):\n return [[\"present\"] for _ in range(n)]", "example_args": [], "origin": "plan", "screen_ratio": 1.1108862711908818, "valid_fraction": 1.0, "error": null }, { "id": "R1", "kind": "suspected_slowdown", "family": "present_string", "why": "The new version checks three length‑based branches before testing `date == \"present\"`. The old version tested the literal string first, so the \"present\" case now does extra work.", "size_meaning": "Number of calls (each call receives the same \"present\" argument).", "make_source": "def make(n):\n # Return n calls each with the string \"present\"\n return [[\"present\"] for _ in range(n)]", "example_args": [ [ "present" ] ], "origin": "archive", "screen_ratio": 1.1108526167919468, "valid_fraction": 1.0, "error": null }, { "id": "F6.1", "kind": "suspected_slowdown", "family": "Literal 'past'", "why": "In the original version the `date == \"present\"` test was performed *before* any regex work, so it returned immediately. The new version first evaluates three `len`/index checks and a full `str.isdigit()` scan (7 characters) before reaching the equality test, adding overhead for this valid input.", "size_meaning": "Number of repetitions; each call processes the same literal.", "make_source": "def make(n):\n return [[\"2024-05-01\"] for _ in range(n)]", "example_args": [], "origin": "expand:F6", "screen_ratio": 0.6422466760725388, "valid_fraction": 1.0, "error": null }, { "id": "F6.2", "kind": "suspected_slowdown", "family": "Future date", "why": "In the original version the `date == \"present\"` test was performed *before* any regex work, so it returned immediately. The new version first evaluates three `len`/index checks and a full `str.isdigit()` scan (7 characters) before reaching the equality test, adding overhead for this valid input.", "size_meaning": "Number of repetitions; each call processes the same literal.", "make_source": "def make(n):\n return [[\"2099-12-31\"] for _ in range(n)]", "example_args": [], "origin": "expand:F6", "screen_ratio": 0.6422836019224843, "valid_fraction": 1.0, "error": null }, { "id": "F6.3", "kind": "suspected_slowdown", "family": "Mid-range year", "why": "In the original version the `date == \"present\"` test was performed *before* any regex work, so it returned immediately. The new version first evaluates three `len`/index checks and a full `str.isdigit()` scan (7 characters) before reaching the equality test, adding overhead for this valid input.", "size_meaning": "Number of repetitions; each call processes the same literal.", "make_source": "def make(n):\n return [[\"2050-06-15\"] for _ in range(n)]}]}, null]}], ", "example_args": [], "origin": "expand:F6", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: syntax error: unmatched '}' (line 3)" }, { "id": "R1.1", "kind": "suspected_slowdown", "family": "empty", "why": "The new version checks three length‑based branches before testing `date == \"present\"`. The old version tested the literal string first, so the \"present\" case now does extra work.", "size_meaning": "Number of calls (each call receives the same \"present\" argument).", "make_source": "def make(n):\n \"\"\"Return n calls with empty string.\"\"\"\n return [[\"\"] for _ in range(n)]}, {", "example_args": [ [ "present" ] ], "origin": "expand:R1", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: syntax error: unmatched '}' (line 4)" } ], "findings": [ { "family_id": "F6", "kind": "suspected_slowdown", "family": "The literal string \"present\"", "why": "In the original version the `date == \"present\"` test was performed *before* any regex work, so it returned immediately. The new version first evaluates three `len`/index checks and a full `str.isdigit()` scan (7 characters) before reaching the equality test, adding overhead for this valid input.", "origin": "plan", "screen_ratio": 1.1108862711908818, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 12242.7325, "head_ir": 13519.98625, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.104327506134762, "judged": true }, { "n": 4, "base_ir": 12065.621875, "head_ir": 13343.883125, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.105942425781514, "judged": true }, { "n": 16, "base_ir": 11951.477213541666, "head_ir": 13276.178385416666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1108399529368673, "judged": true }, { "n": 64, "base_ir": 11909.63671875, "head_ir": 13186.938802083334, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072494580227126, "judged": true }, { "n": 256, "base_ir": 11909.795247395834, "head_ir": 13186.87109375, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072290345741593, "judged": true }, { "n": 1024, "base_ir": 11909.859049479166, "head_ir": 13186.985026041666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072326692748182, "judged": true }, { "n": 4096, "base_ir": 11909.9052734375, "head_ir": 13186.948893229166, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072253381090982, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.1108399529368673, "worst_search": null }, { "family_id": "R1", "kind": "suspected_slowdown", "family": "present_string", "why": "The new version checks three length‑based branches before testing `date == \"present\"`. The old version tested the literal string first, so the \"present\" case now does extra work.", "origin": "archive", "screen_ratio": 1.1108526167919468, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 12242.87875, "head_ir": 13521.29625, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.104421315125742, "judged": true }, { "n": 4, "base_ir": 12065.51375, "head_ir": 13344.14, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1059736266928542, "judged": true }, { "n": 16, "base_ir": 11951.315104166666, "head_ir": 13276.30078125, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1108652617335306, "judged": true }, { "n": 64, "base_ir": 11909.7685546875, "head_ir": 13187.216471354166, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072605156683655, "judged": true }, { "n": 256, "base_ir": 11909.525065104166, "head_ir": 13187.24609375, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072856408346339, "judged": true }, { "n": 1024, "base_ir": 11909.753580729166, "head_ir": 13186.957682291666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072401786405646, "judged": true }, { "n": 4096, "base_ir": 11909.896484375, "head_ir": 13186.980794270834, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.107228833732626, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.1108652617335306, "worst_search": null }, { "family_id": "F5", "kind": "suspected_slowdown", "family": "present-string", "why": "In the new code the literal \"present\" is tested *after* the three length‑/index‑checks (len==10, len==7, len==4). The old version checked `date == \"present\"` immediately after the integer case, so the new version does extra Python work for every \"present\" input.", "origin": "plan", "screen_ratio": 1.110830555322604, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 12241.98125, "head_ir": 13520.05, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1044004825607783, "judged": true }, { "n": 4, "base_ir": 12065.523125, "head_ir": 13343.816875, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.105945986490329, "judged": true }, { "n": 16, "base_ir": 11951.564778645834, "head_ir": 13275.80859375, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1108008733275014, "judged": true }, { "n": 64, "base_ir": 11909.781901041666, "head_ir": 13186.899088541666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072326259298082, "judged": true }, { "n": 256, "base_ir": 11909.89453125, "head_ir": 13186.89453125, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072217723380606, "judged": true }, { "n": 1024, "base_ir": 11909.617838541666, "head_ir": 13186.8935546875, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072474141036113, "judged": true }, { "n": 4096, "base_ir": 11909.729166666666, "head_ir": 13187.036458333334, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.107249062828535, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.1108008733275014, "worst_search": null }, { "family_id": "F4", "kind": "suspected_slowdown", "family": "present_literal", "why": "In the original version the literal string \"present\" was tested *before* any regex match, so it returned immediately. The new version first checks the length‑10, length‑7 and length‑4 patterns, then finally tests `date == \"present\"`. Each call therefore performs three cheap length/index checks that the old code skipped.", "origin": "plan", "screen_ratio": 1.1108182414181655, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 12242.4, "head_ir": 13521.0175, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.104441735280664, "judged": true }, { "n": 4, "base_ir": 12065.620625, "head_ir": 13344.01125, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.105953159371775, "judged": true }, { "n": 16, "base_ir": 11951.082356770834, "head_ir": 13276.126627604166, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1108723236337363, "judged": true }, { "n": 64, "base_ir": 11909.800455729166, "head_ir": 13187.060221354166, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072444303641191, "judged": true }, { "n": 256, "base_ir": 11909.543294270834, "head_ir": 13186.988932291666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072623530942078, "judged": true }, { "n": 1024, "base_ir": 11909.875325520834, "head_ir": 13186.8525390625, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.1072200320019574, "judged": true }, { "n": 4096, "base_ir": 11909.63671875, "head_ir": 13186.990559895834, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 1.107253803899394, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "slower", "worst_ratio": 1.1108723236337363, "worst_search": null }, { "family_id": "F3", "kind": "claimed_speedup", "family": "ISO full dates (YYYY-MM-DD)", "why": "The new code uses simple length and character checks instead of three pre‑compiled regex fullmatches. For well‑formed ISO dates the fast path is a single `len` test and two index checks, which is cheaper than the regex engine.", "origin": "plan", "screen_ratio": 0.6422891505593947, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 6356.20125, "head_ir": 4199.215, "input_chars": 14.0, "valid_fraction": 1.0, "ratio": 0.6606485280811397, "judged": true }, { "n": 4, "base_ir": 6190.17375, "head_ir": 4033.408125, "input_chars": 14.0, "valid_fraction": 1.0, "ratio": 0.6515823768274679, "judged": true }, { "n": 16, "base_ir": 6030.602163461538, "head_ir": 3873.422275641026, "input_chars": 14.0, "valid_fraction": 1.0, "ratio": 0.6422944460023374, "judged": true }, { "n": 64, "base_ir": 6004.7874177631575, "head_ir": 3848.1792763157896, "input_chars": 14.0, "valid_fraction": 1.0, "ratio": 0.6408518751108885, "judged": true }, { "n": 256, "base_ir": 6004.90625, "head_ir": 3847.6916118421054, "input_chars": 14.0, "valid_fraction": 1.0, "ratio": 0.6407579821653511, "judged": true }, { "n": 1024, "base_ir": 6005.108963815789, "head_ir": 3847.8129111842104, "input_chars": 14.0, "valid_fraction": 1.0, "ratio": 0.6407565515246235, "judged": true }, { "n": 4096, "base_ir": 6004.785361842105, "head_ir": 3847.5596217105262, "input_chars": 14.0, "valid_fraction": 1.0, "ratio": 0.6407489010615026, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [ { "args": [ "4237594594" ], "cpu_ratio": 0.593, "base_ir": 11526.0625, "head_ir": 6665.0725, "ir_ratio": 0.57826100630636, "outcome": "exc:builtins.ValueError" }, { "args": [ "1045093214" ], "cpu_ratio": 0.593, "base_ir": 11526.6425, "head_ir": 6666.6775, "ir_ratio": 0.5783711518770536, "outcome": "exc:builtins.ValueError" } ], "error": null, "verdict": "faster", "worst_ratio": 0.6606485280811397, "worst_search": { "args": [ "1045093214" ], "cpu_ratio": 0.593, "base_ir": 11526.6425, "head_ir": 6666.6775, "ir_ratio": 0.5783711518770536, "outcome": "exc:builtins.ValueError" } } ], "seconds": { "plan": 12.73, "screen": 497.28, "full": 752.98, "total": 1262.98 } } } } ] }, { "id": "rendercv#19", "pr": 19, "title": "⚡️ Speed up function `validate_start_and_end_date_fields` by 34%", "base": "7f7abe4ebf939e25ebac92b9d1d43f226e80b373", "head": "9ec5e7bd90913bce5b157e34b11a6e797fe57164", "merged": "2026-08-10", "after_ultra_release": true, "known": [], "repo": "https://github.com/IanHollow/rendercv", "install": { "exclude_newer": "2026-08-11" }, "targets": [ { "target": "rendercv.data.models.computers:get_date_object", "run_id": "bt-rendercv#19-592e6c", "image": "5d1921f3-ef7b-4ff5-9028-f41aa6b1263d", "backend": "contree", "usd": 0.1345, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 13.3, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 8.6, "error": null }, "breakwater": { "classes": [], "known": [], "seconds": 60.4, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 0.0, "error": null } }, "findings": [], "perf": { "slower_families": [], "worst_valid_ratio": null, "worst_family": null, "speedup_ratio_small": 0.4595458046346701, "report": { "run_id": "bt-rendercv#19-592e6c", "target": "rendercv.data.models.computers:get_date_object", "image": "5d1921f3-ef7b-4ff5-9028-f41aa6b1263d", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "present-strings", "why": "The new version checks for the literal string \"present\" before any regex matching, turning a three‑regex‑miss path into a single cheap equality test.", "size_meaning": "Number of calls (each call passes the same \"present\" value).", "make_source": "def make(n):\n return [[\"present\"] for _ in range(n)]", "example_args": [], "origin": "plan", "screen_ratio": 0.46382724575039547, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "Well‑formed YYYY‑MM‑DD dates (e.g. \"2023-07-15\")", "why": "The new code uses pre‑compiled regex objects (_YMD_PATTERN, _YM_PATTERN, _Y_PATTERN) instead of compiling the patterns on every call with `re.fullmatch`. This removes the per‑call compilation overhead and makes the fast path noticeably quicker.", "size_meaning": "Number of generated date strings; each string is a separate call to `get_date_object`.", "make_source": "\ndef make(n):\n # generate n distinct YYYY-MM-DD strings\n base_year = 2000\n return [[f\"{base_year + i:04d}-01-01\"] for i in range(n)]", "example_args": [ [ "2023-01-01" ], [ "2024-02-02" ] ], "origin": "plan", "screen_ratio": 0.6830554884651924, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "Valid full dates (YYYY‑MM‑DD) – the new pre‑compiled regex should be faster than the old inline patterns.", "why": "The head version compiles the three patterns once at import time and checks them with the fast `fullmatch` method, while the base version recompiles the same patterns on every call. For a batch of valid YMD strings the head path avoids repeated compilation overhead.", "size_meaning": "n = number of calls (1‑8). Cost grows linearly with n because each call parses a date.", "make_source": "def make(n):\n \"\"\"Return up to 8 calls with distinct valid YMD dates.\"\"\"\n n = max(1, min(n, 8))\n return [[f\"{2000 + i}-01-01\"] for i in range(n)]", "example_args": [ [ "2000-01-01" ], [ "2001-01-01" ], [ "2002-01-01" ] ], "origin": "plan", "screen_ratio": 0.6972501248675455, "valid_fraction": 1.0, "error": null }, { "id": "F4", "kind": "suspected_slowdown", "family": "year-only-strings", "why": "In the old code a \"YYYY\" string matched the fourth condition (re.fullmatch(r\"\\\\d{4}\", date)). The new code evaluates three extra cheap branches (int, \"present\", YMD, YM) before reaching the compiled YYYY pattern, adding constant overhead per call.", "size_meaning": "Number of distinct year strings processed.", "make_source": "def make(n):\n base = 2000\n return [[str(base + i)] for i in range(n)]", "example_args": [], "origin": "plan", "screen_ratio": 0.5295285807468356, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "claimed_speedup", "family": "Well‑formed YYYY‑MM dates (e.g. \"2023-07\")", "why": "Same reason as the YYYY‑MM‑DD case – the compiled `_YM_PATTERN` avoids per‑call regex compilation.", "size_meaning": "Number of generated YYYY‑MM strings.", "make_source": "def make(n):\n base_year = 2000\n return [[f\"{base_year + i:04d}-01\"] for i in range(n)]", "example_args": [ [ "2023-01" ], [ "2024-02" ] ], "origin": "plan", "screen_ratio": 0.598387389341889, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "Invalid long strings – both versions must try all three regexes before raising ValueError, so the head version does not gain any advantage and pays the same three full‑match scans.", "why": "The change only reorders the checks and pre‑compiles the patterns. For inputs that match none of the patterns (e.g. a long run of ‘x’ characters) the function still executes three `fullmatch` calls on a string of length n. The extra early `date == \"present\"` equality test is O(1) and does not offset the three scans.", "size_meaning": "n = length of the bogus string. Cost grows with n because each regex scans the whole string.", "make_source": "def make(n):\n \"\"\"Return a single call with an invalid string of length n.\"\"\"\n n = max(1, n)\n return [[\"x\" * n]]", "example_args": [ [ "xxxxxxxx" ], [ "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" ] ], "origin": "plan", "screen_ratio": 0.5542066228255721, "valid_fraction": 0.0, "error": null }, { "id": "R1", "kind": "suspected_slowdown", "family": "present_string", "why": "The new version checks three length‑based branches before testing `date == \"present\"`. The old version tested the literal string first, so the \"present\" case now does extra work.", "size_meaning": "Number of calls (each call receives the same \"present\" argument).", "make_source": "def make(n):\n # Return n calls each with the string \"present\"\n return [[\"present\"] for _ in range(n)]", "example_args": [ [ "present" ] ], "origin": "archive", "screen_ratio": 0.46383058998638066, "valid_fraction": 1.0, "error": null } ], "findings": [ { "family_id": "F1", "kind": "claimed_speedup", "family": "present-strings", "why": "The new version checks for the literal string \"present\" before any regex matching, turning a three‑regex‑miss path into a single cheap equality test.", "origin": "plan", "screen_ratio": 0.46382724575039547, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 26247.85375, "head_ir": 12242.20625, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.46640789630275964, "judged": true }, { "n": 4, "base_ir": 26071.800625, "head_ir": 12065.9, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.4627950394968165, "judged": true }, { "n": 16, "base_ir": 25852.288736979168, "head_ir": 11991.164388020834, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.4638337638117374, "judged": true }, { "n": 64, "base_ir": 25916.229817708332, "head_ir": 11909.9990234375, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.4595575478073397, "judged": true }, { "n": 256, "base_ir": 25916.156575520832, "head_ir": 11909.768229166666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.45954994115200193, "judged": true }, { "n": 1024, "base_ir": 25916.212565104168, "head_ir": 11909.699869791666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.45954631062981466, "judged": true }, { "n": 4096, "base_ir": 25916.2666015625, "head_ir": 11909.711588541666, "input_chars": 11.0, "valid_fraction": 1.0, "ratio": 0.4595458046346701, "judged": true } ], "slope_base": null, "slope_head": null, "top_inputs": [], "error": null, "verdict": "faster", "worst_ratio": 0.46640789630275964, "worst_search": null } ], "seconds": { "plan": 10.45, "screen": 326.18, "full": 249.2, "total": 585.84 } } } } ] }, { "id": "json_repair#113", "pr": 113, "title": "⚡️ Speed up method `ObjectComparer.is_same_object` by 109%", "base": "ec462b47b30464b66eaaec6753a8db3d79522253", "head": "cbfa7a1c6fa8360f90303c5b9b8054ef2f6298bb", "merged": "2025-05-16", "after_ultra_release": false, "known": [], "repo": "https://github.com/mangiucugna/json_repair", "install": { "exclude_newer": "2025-05-17" }, "targets": [ { "target": "json_repair.object_comparer:ObjectComparer.is_same_object", "run_id": "bt-json_repair#113-ba5cd1", "image": "44ce7ab3-2490-4df1-96bc-08c7d76584d1", "backend": "contree", "usd": 0.2319, "harvested": 0, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 9.2, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 4.8, "error": null }, "breakwater": { "classes": [], "known": [], "seconds": 64.7, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 0.0, "error": null } }, "findings": [], "perf": { "slower_families": [ { "id": "R1", "family": "large dicts with identical length but one key swapped, and deep equal values on the shared keys", "verdict": "complexity-regression", "ladder": [ 1.2602686492869575, 4.103154504112442, 30.477666763689747, 160.05565269193215 ], "origin": "archive" }, { "id": "R2", "family": "swap position of the mismatched key within dict ordering", "verdict": "complexity-regression", "ladder": [ 0.35832971330687147, 1.5079980646476456, 14.324632030247788, 79.2704114307474, 363.31034819644236 ], "origin": "archive" }, { "id": "R3", "family": "Large dictionaries with same length but almost no common keys", "verdict": "complexity-regression", "ladder": [ 0.6398499950965152, 1.068620960906729, 2.199936083248765, 3.000317692940644, 3.4632837101604963, 3.664158315908098, 3.7265635923582434 ], "origin": "archive" }, { "id": "R4", "family": "Dicts with numeric string keys where only the first matches", "verdict": "complexity-regression", "ladder": [ 0.3560818082832093, 1.0796623428018193, 1.3734950008795683, 2.176923134283045, 1.3707106923900108, 3.389626599822074, 1.291599474451235 ], "origin": "archive" } ], "worst_valid_ratio": 363.31034819644236, "worst_family": "swap position of the mismatched key within dict ordering", "speedup_ratio_small": 0.5611763686241454, "report": { "run_id": "bt-json_repair#113-ba5cd1", "target": "json_repair.object_comparer:ObjectComparer.is_same_object", "image": "44ce7ab3-2490-4df1-96bc-08c7d76584d1", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "large flat dicts with many keys", "why": "The old code built two key‑sets, computed intersections and differences (O(N) allocations) before iterating. The new code only checks length and then iterates over obj1's keys, doing a single hash lookup per key. For a dict with many keys the allocation‑heavy set work dominates, so the new version is noticeably faster.", "size_meaning": "n = number of keys in each dict (values are tiny atoms)", "make_source": "\ndef make(n):\n # build two identical dicts with n string keys and simple int values\n keys = [f'k{i}' for i in range(n)]\n d1 = {k: i for i, k in enumerate(keys)}\n d2 = {k: i for i, k in enumerate(keys)}\n return [[d1, d2]]", "example_args": [ [ { "k0": 0, "k1": 1, "k2": 2 } ], [ { "k0": 0, "k1": 1, "k2": 2 } ] ], "origin": "plan", "screen_ratio": 0.6220240374148894, "valid_fraction": 1.0, "error": null }, { "id": "F2", "kind": "claimed_speedup", "family": "Large dictionaries with identical key sets and simple values", "why": "The new version avoids building two Python `set` objects, computing intersections and differences, and building path strings for recursive calls. For large equal dictionaries the old code spends O(n) time in C‑level set construction plus a Python loop over the common keys; the new code does a single length check and a plain Python loop over `obj1` keys, which is measurably faster.", "size_meaning": "n = number of keys in each dictionary", "make_source": "def make(n):\n # create two identical dicts with n string keys mapping to small ints\n d = {f'k{i}': i for i in range(n)}\n return [(d, d)]", "example_args": [ [ { "obj1": { "k0": 0, "k1": 1, "k2": 2 }, "obj2": { "k0": 0, "k1": 1, "k2": 2 } } ] ], "origin": "plan", "screen_ratio": 0.6220240374148894, "valid_fraction": 1.0, "error": null }, { "id": "F3", "kind": "claimed_speedup", "family": "deep_nesting", "why": "Old code built a `path` string at every recursive call (`f\"{path}/{key}\"` and `f\"{path}[{i}]\"`). For deeply nested structures this creates O(depth²) temporary strings, while the new version drops the `path` argument entirely. The speed‑up grows with nesting depth.", "size_meaning": "nesting depth", "make_source": "def make(n):\n # build a nested list of depth n, e.g. [[[...]]]\n obj = []\n for _ in range(n):\n obj = [obj]\n # both arguments are identical, so the function must walk the whole structure\n return [[obj, obj]]", "example_args": [ [ [ [ [] ] ] ], [ [ [ [] ] ] ] ], "origin": "plan", "screen_ratio": null, "valid_fraction": null, "error": "exit 1: ks\n File \"/usr/local/lib/python3.12/json/encoder.py\", line 326, in _iterencode_list\n yield from chunks\n [Previous line repeated 990 more times]\n File \"/usr/local/lib/python3.12/json/encoder.py\", line 321, in _iterencode_list\n chunks = _iterencode_list(value, _current_indent_level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nRecursionError: maximum recursion depth exceeded\n" }, { "id": "F4", "kind": "claimed_speedup", "family": "large flat lists", "why": "The old implementation computed min(len) and then looped up to that length before finally comparing the lengths. The new version checks the lengths first and returns immediately on mismatch, avoiding the needless element‑wise loop when the lengths differ.", "size_meaning": "n = length of each list (elements are tiny atoms)", "make_source": "def make(n):\n l1 = list(range(n))\n l2 = list(range(n))\n return [[l1, l2]]", "example_args": [ [ 0, 1, 2 ], [ 0, 1, 2 ] ], "origin": "plan", "screen_ratio": 0.5676453099407834, "valid_fraction": 1.0, "error": null }, { "id": "F5", "kind": "suspected_slowdown", "family": "Dictionaries where `obj2` lacks the last key of `obj1` (all other keys identical)", "why": "The old implementation builds two `set`s and computes `extra_keys1 = keys1 - keys2` in C, which discovers the missing key without a Python‑level loop over all keys. The new implementation iterates over every key of `obj1` in pure Python and only fails on the final iteration, so the Python loop runs `n` times instead of the old version’s fast C‑level set difference.", "size_meaning": "n = number of keys in `obj1` (the missing key is the nth)", "make_source": "def make(n):\n # obj1 has keys k0..k{n-1}, obj2 missing the last one\n obj1 = {f'k{i}': i for i in range(n)}\n obj2 = {f'k{i}': i for i in range(n-1)}\n return [(obj1, obj2)]", "example_args": [ [ { "obj1": { "k0": 0, "k1": 1, "k2": 2 }, "obj2": { "k0": 0, "k1": 1 } } ] ], "origin": "plan", "screen_ratio": 0.1258598681291185, "valid_fraction": 1.0, "error": null }, { "id": "F6", "kind": "suspected_slowdown", "family": "large_flat_dict", "why": "The new implementation iterates over `obj1` keys in pure Python (`for key in obj1: …`). The old version created two `set` objects and used the C‑level intersection (`keys1 & keys2`) before a Python loop over the (usually much smaller) `common_keys`. For very large dictionaries with many keys the extra Python‑level loop can be slower even though both algorithms are O(N).", "size_meaning": "number of keys", "make_source": "def make(n):\n # a dict with n distinct string keys, all mapping to the same simple value\n d = {f'k{i}': i for i in range(n)}\n return [[d, d]]", "example_args": [ [ { "k0": 0, "k1": 1, "k2": 2 } ], [ { "k0": 0, "k1": 1, "k2": 2 } ] ], "origin": "plan", "screen_ratio": 0.6220240374148894, "valid_fraction": 1.0, "error": null }, { "id": "R1", "kind": "suspected_slowdown", "family": "large dicts with identical length but one key swapped, and deep equal values on the shared keys", "why": "Head iterates over obj1 keys in Python, recursing into each shared value before discovering the missing key; base builds key sets in C and detects the extra/missing keys via set difference, returning False before any value recursion.", "size_meaning": "n is the number of keys per dict (>=2). Shared keys = n‑1, each holding a list of length n to make recursion costly.", "make_source": "def make(n):\n # shared keys k0..k{n-2} map to identical large lists\n shared = {f'k{i}': list(range(n)) for i in range(n-1)}\n # each side has its own extra key\n obj1 = dict(shared)\n obj1[f'extra1_{n}'] = list(range(n))\n obj2 = dict(shared)\n obj2[f'extra2_{n}'] = list(range(n))\n # both dicts have length n\n return [[obj1, obj2]] * 4", "example_args": [], "origin": "archive", "screen_ratio": 30.477666763689747, "valid_fraction": 1.0, "error": "ladder inputs exceed 2000000 chars at n=256; ladder stops here" }, { "id": "R2", "kind": "suspected_slowdown", "family": "swap position of the mismatched key within dict ordering", "why": "Head iterates over obj1 keys in Python, recursing into each shared value before discovering the missing key; base builds key sets in C and detects the extra/missing keys via set difference, returning False before any value recursion.", "size_meaning": "n is the number of keys per dict (>=2). Shared keys = n‑1, each holding a list of length n to make recursion costly.", "make_source": "\ndef make(n):\n shared_val = list(range(n))\n prefixed_key = f'{chr(97 + n % 26)}key'\n other_keys = [f'{chr(97+i%26)}{i}' for i in range(n - 1)]\n all_keys = other_keys[:]\n idx = len(all_keys)//2\n all_keys.insert(idx, prefixed_key)\n o1 = {}\n for k in all_keys:\n val = shared_val if k != prefixed_key else list(range(n+1))\n o1[k] = val\n all_keys.remove(prefixed_key)\n alt_key = f'zzz_{n}'\n all_keys.append(alt_key)\n o2 = {}\n for k in all_keys:\n val = shared_val if k != alt_key else list(range(n+1))\n o2[k] = val\n return [[o1, o2]]", "example_args": [], "origin": "archive", "screen_ratio": 14.324632030247788, "valid_fraction": 1.0, "error": "ladder inputs exceed 2000000 chars at n=1024; ladder stops here" }, { "id": "R3", "kind": "suspected_slowdown", "family": "Large dictionaries with same length but almost no common keys", "why": "The old implementation built two key‑sets in C, computed the intersection once, and then iterated only over the (tiny) common set. The new code iterates over *all* keys of the first dict in pure Python, performing a `key not in obj2` test for each one. When the overlap is tiny the new loop does O(n) Python work while the old path does O(n) C work plus O(1) Python work.", "size_meaning": "n = number of keys in each dictionary (both have length n, only 1 key overlaps)", "make_source": "def make(n):\n # dict1: keys k0..k{n-1}\n # dict2: keys k0..k{n-2} plus 'extra' -> same length, 1 common key\n d1 = {f'k{i}': i for i in range(n)}\n d2 = {f'k{i}': i for i in range(n-1)}\n d2['extra'] = n\n return [[d1, d2]]", "example_args": [ [ {} ], [ {} ] ], "origin": "archive", "screen_ratio": 2.199936083248765, "valid_fraction": 1.0, "error": null }, { "id": "R4", "kind": "suspected_slowdown", "family": "Dicts with numeric string keys where only the first matches", "why": "The old implementation built two key‑sets in C, computed the intersection once, and then iterated only over the (tiny) common set. The new code iterates over *all* keys of the first dict in pure Python, performing a `key not in obj2` test for each one. When the overlap is tiny the new loop does O(n) Python work while the old path does O(n) C work plus O(1) Python work.", "size_meaning": "n = number of keys in each dictionary (both have length n, only 1 key overlaps)", "make_source": "def make(n):\n seq_a = [] ; running_total = 0\n for idx in range(n):\n element_value = idx * 17 + 42\n seq_a.append((f'num_{idx:05d}', element_value))\n running_total += element_value\n seq_b = [] ; accumulator = 0\n for idx in range(n - 1):\n element_value = idx * 17 + 42\n seq_b.append((f'num_{idx:05d}', element_value))\n accumulator += element_value\n unique_val = running_total + accumulator + 1\n seq_b.append((f'special_{unique_val}', unique_val))\n random.shuffle(seq_a)\n random.shuffle(seq_b)\n dict_a = dict(seq_a)\n dict_b = dict(seq_b)\n verification_sum_a = sum(dict_a.values())\n verification_sum_b = sum(dict_b.values())\n if verification_sum_a != running_total:\n pass\n if verification_sum_b != (running_total - ((n-2)*17+42 if n>=2 else 0) + unique_val):\n pass\n shuffle_attempts = 0\n while shuffle_attempts < 3:\n temp_list_a = list(dict_a.items())\n random.shuffle(temp_list_a)\n dict_a = dict(temp_list_a)\n temp_list_b = list(dict_b.items())\n random.shuffle(temp_list_b)\n dict_b = dict(temp_list_b)\n shuffle_attempts += 1\n final_dict_a = {} ; final_dict_b = {}\n for key_val, value_val in dict_a.items():\n final_dict_a[key_val] = value_val\n for key_val, value_val in dict_b.items():\n final_dict_b[key_val] = value_val\n return [[final_dict_a, final_dict_b]]", "example_args": [ [ {} ], [ {} ] ], "origin": "archive", "screen_ratio": 1.3734950008795683, "valid_fraction": 1.0, "error": null }, { "id": "R2.1", "kind": "suspected_slowdown", "family": "swap position of mismatch at front", "why": "Head iterates over obj1 keys in Python, recursing into each shared value before discovering the missing key; base builds key sets in C and detects the extra/missing keys via set difference, returning False before any value recursion.", "size_meaning": "n is the number of keys per dict (>=2). Shared keys = n‑1, each holding a list of length n to make recursion costly.", "make_source": "", "example_args": [], "origin": "expand:R2", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: code must assign `strategy`" }, { "id": "R2.2", "kind": "suspected_slowdown", "family": "swap position of mismatch near end", "why": "Head iterates over obj1 keys in Python, recursing into each shared value before discovering the missing key; base builds key sets in C and detects the extra/missing keys via set difference, returning False before any value recursion.", "size_meaning": "n is the number of keys per dict (>=2). Shared keys = n‑1, each holding a list of length n to make recursion costly.", "make_source": "", "example_args": [], "origin": "expand:R2", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: code must assign `strategy`" }, { "id": "R2.3", "kind": "suspected_slowdown", "family": "preserve prefix but reposition deeper nesting under first mismatched key", "why": "Head iterates over obj1 keys in Python, recursing into each shared value before discovering the missing key; base builds key sets in C and detects the extra/missing keys via set difference, returning False before any value recursion.", "size_meaning": "n is the number of keys per dict (>=2). Shared keys = n‑1, each holding a list of length n to make recursion costly.", "make_source": "", "example_args": [], "origin": "expand:R2", "screen_ratio": null, "valid_fraction": null, "error": "make(n) rejected: code must assign `strategy`" } ], "findings": [ { "family_id": "R1", "kind": "suspected_slowdown", "family": "large dicts with identical length but one key swapped, and deep equal values on the shared keys", "why": "Head iterates over obj1 keys in Python, recursing into each shared value before discovering the missing key; base builds key sets in C and detects the extra/missing keys via set difference, returning False before any value recursion.", "origin": "archive", "screen_ratio": 30.477666763689747, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 9040.25, "head_ir": 11393.143656716418, "input_chars": 72.0, "valid_fraction": 1.0, "ratio": 1.2602686492869575, "judged": true }, { "n": 4, "base_ir": 10217.25, "head_ir": 41922.955357142855, "input_chars": 176.0, "valid_fraction": 1.0, "ratio": 4.103154504112442, "judged": true }, { "n": 16, "base_ir": 19754.75, "head_ir": 602078.6875, "input_chars": 2012.0, "valid_fraction": 1.0, "ratio": 30.477666763689747, "judged": true }, { "n": 64, "base_ir": 58122.75, "head_ir": 9302874.6875, "input_chars": 32636.0, "valid_fraction": 1.0, "ratio": 160.05565269193215, "judged": true } ], "slope_base": 0.3078025037545104, "slope_head": 1.082475659224245, "top_inputs": [], "error": "ladder inputs exceed 2000000 chars at n=256; ladder stops here", "verdict": "complexity-regression", "worst_ratio": 160.05565269193215, "worst_search": null }, { "family_id": "R2", "kind": "suspected_slowdown", "family": "swap position of the mismatched key within dict ordering", "why": "Head iterates over obj1 keys in Python, recursing into each shared value before discovering the missing key; base builds key sets in C and detects the extra/missing keys via set difference, returning False before any value recursion.", "origin": "archive", "screen_ratio": 14.324632030247788, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 9639.6625, "head_ir": 3454.1775, "input_chars": 71.0, "valid_fraction": 1.0, "ratio": 0.35832971330687147, "judged": true }, { "n": 4, "base_ir": 10906.098214285714, "head_ir": 16446.375, "input_chars": 175.0, "valid_fraction": 1.0, "ratio": 1.5079980646476456, "judged": true }, { "n": 16, "base_ir": 19748.0, "head_ir": 282882.8333333333, "input_chars": 2012.0, "valid_fraction": 1.0, "ratio": 14.324632030247788, "judged": true }, { "n": 64, "base_ir": 57774.0, "head_ir": 4579768.75, "input_chars": 32636.0, "valid_fraction": 1.0, "ratio": 79.2704114307474, "judged": true }, { "n": 256, "base_ir": 202156.0, "head_ir": 73445366.75, "input_chars": 603957.0, "valid_fraction": 1.0, "ratio": 363.31034819644236, "judged": true } ], "slope_base": 0.3389646045011183, "slope_head": 1.0816434987267658, "top_inputs": [], "error": "ladder inputs exceed 2000000 chars at n=1024; ladder stops here", "verdict": "complexity-regression", "worst_ratio": 363.31034819644236, "worst_search": null }, { "family_id": "R3", "kind": "suspected_slowdown", "family": "Large dictionaries with same length but almost no common keys", "why": "The old implementation built two key‑sets in C, computed the intersection once, and then iterated only over the (tiny) common set. The new code iterates over *all* keys of the first dict in pure Python, performing a `key not in obj2` test for each one. When the overlap is tiny the new loop does O(n) Python work while the old path does O(n) C work plus O(1) Python work.", "origin": "archive", "screen_ratio": 2.199936083248765, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 9585.02, "head_ir": 6132.975, "input_chars": 43.0, "valid_fraction": 1.0, "ratio": 0.6398499950965152, "judged": true }, { "n": 4, "base_ir": 10831.02, "head_ir": 11574.255, "input_chars": 79.0, "valid_fraction": 1.0, "ratio": 1.068620960906729, "judged": true }, { "n": 16, "base_ir": 20137.08064516129, "head_ir": 44300.290322580644, "input_chars": 318.0, "valid_fraction": 1.0, "ratio": 2.199936083248765, "judged": true }, { "n": 64, "base_ir": 58682.0, "head_ir": 176064.64285714287, "input_chars": 1374.0, "valid_fraction": 1.0, "ratio": 3.000317692940644, "judged": true }, { "n": 256, "base_ir": 203556.33333333334, "head_ir": 704973.3333333334, "input_chars": 6221.0, "valid_fraction": 1.0, "ratio": 3.4632837101604963, "judged": true }, { "n": 1024, "base_ir": 773782.0, "head_ir": 2835259.75, "input_chars": 26284.0, "valid_fraction": 1.0, "ratio": 3.664158315908098, "judged": true }, { "n": 4096, "base_ir": 3046734.0, "head_ir": 11353848.0, "input_chars": 118444.0, "valid_fraction": 1.0, "ratio": 3.7265635923582434, "judged": true } ], "slope_base": 0.7408538196302779, "slope_head": 0.9467405643569164, "top_inputs": [], "error": null, "verdict": "complexity-regression", "worst_ratio": 3.7265635923582434, "worst_search": null }, { "family_id": "R4", "kind": "suspected_slowdown", "family": "Dicts with numeric string keys where only the first matches", "why": "The old implementation built two key‑sets in C, computed the intersection once, and then iterated only over the (tiny) common set. The new code iterates over *all* keys of the first dict in pure Python, performing a `key not in obj2` test for each one. When the overlap is tiny the new loop does O(n) Python work while the old path does O(n) C work plus O(1) Python work.", "origin": "archive", "screen_ratio": 1.3734950008795683, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 9533.02, "head_ir": 3394.535, "input_chars": 75.0, "valid_fraction": 1.0, "ratio": 0.3560818082832093, "judged": true }, { "n": 4, "base_ir": 10679.42700729927, "head_ir": 11530.175182481751, "input_chars": 143.0, "valid_fraction": 1.0, "ratio": 1.0796623428018193, "judged": true }, { "n": 16, "base_ir": 20397.70588235294, "head_ir": 28016.147058823528, "input_chars": 576.0, "valid_fraction": 1.0, "ratio": 1.3734950008795683, "judged": true }, { "n": 64, "base_ir": 58671.3125, "head_ir": 127722.9375, "input_chars": 2319.0, "valid_fraction": 1.0, "ratio": 2.176923134283045, "judged": true }, { "n": 256, "base_ir": 199974.0, "head_ir": 274106.5, "input_chars": 9619.0, "valid_fraction": 1.0, "ratio": 1.3707106923900108, "judged": true }, { "n": 1024, "base_ir": 776727.0, "head_ir": 2632814.5, "input_chars": 39680.0, "valid_fraction": 1.0, "ratio": 3.389626599822074, "judged": true }, { "n": 4096, "base_ir": 3052809.0, "head_ir": 3943006.5, "input_chars": 162562.0, "valid_fraction": 1.0, "ratio": 1.291599474451235, "judged": true } ], "slope_base": 0.7658703767691176, "slope_head": 0.9202113086524708, "top_inputs": [], "error": null, "verdict": "complexity-regression", "worst_ratio": 3.389626599822074, "worst_search": null }, { "family_id": "F4", "kind": "claimed_speedup", "family": "large flat lists", "why": "The old implementation computed min(len) and then looped up to that length before finally comparing the lengths. The new version checks the lengths first and returns immediately on mismatch, avoiding the needless element‑wise loop when the lengths differ.", "origin": "plan", "screen_ratio": 0.5676453099407834, "valid_fraction": 1.0, "ladder": [ { "n": 2, "base_ir": 14974.915, "head_ir": 8709.815, "input_chars": 16.0, "valid_fraction": 1.0, "ratio": 0.5816270075656523, "judged": true }, { "n": 4, "base_ir": 22900.915, "head_ir": 13195.815, "input_chars": 28.0, "valid_fraction": 1.0, "ratio": 0.5762134395066747, "judged": true }, { "n": 16, "base_ir": 70546.84, "head_ir": 40045.58285714286, "input_chars": 112.0, "valid_fraction": 1.0, "ratio": 0.5676453099407834, "judged": true }, { "n": 64, "base_ir": 261927.9, "head_ir": 147617.15, "input_chars": 496.0, "valid_fraction": 1.0, "ratio": 0.5635793285098686, "judged": true }, { "n": 256, "base_ir": 1030495.4375, "head_ir": 578289.6875, "input_chars": 2344.0, "valid_fraction": 1.0, "ratio": 0.5611763686241454, "judged": true }, { "n": 1024, "base_ir": 4281401.0, "head_ir": 2470474.0, "input_chars": 10072.0, "valid_fraction": 1.0, "ratio": 0.5770246701955738, "judged": true }, { "n": 4096, "base_ir": 17359220.75, "head_ir": 10037026.0, "input_chars": 46936.0, "valid_fraction": 1.0, "ratio": 0.5781956543181813, "judged": true } ], "slope_base": 0.8870050452102738, "slope_head": 0.8866042043131613, "top_inputs": [ { "args": [ [ 61, 9, 54, 2, 0, 71, 35 ], [ 61, 9, 54, 9, 0, 71, 35 ] ], "cpu_ratio": 0.607, "base_ir": 34746.86, "head_ir": 19881.76, "ir_ratio": 0.5721886812218427, "outcome": "ok" }, { "args": [ [ 61, 9, 54, 2, 0, 71, 35 ], [ 61, 9, 54, 2, 0, 71, 35 ] ], "cpu_ratio": 0.607, "base_ir": 34746.86, "head_ir": 19881.76, "ir_ratio": 0.5721886812218427, "outcome": "ok" } ], "error": null, "verdict": "faster", "worst_ratio": 0.5816270075656523, "worst_search": { "args": [ [ 61, 9, 54, 2, 0, 71, 35 ], [ 61, 9, 54, 9, 0, 71, 35 ] ], "cpu_ratio": 0.607, "base_ir": 34746.86, "head_ir": 19881.76, "ir_ratio": 0.5721886812218427, "outcome": "ok" } } ], "seconds": { "plan": 13.79, "screen": 130.28, "full": 180.23, "total": 324.31 } } } } ] }, { "id": "pyhs3#154", "pr": 154, "title": "perf: Speed up function `parse_expression` by 785%", "base": "6ed3734a99ae4d14758d3e6c9c79dfdfc28b3279", "head": "483202e2a159a3e4e642b0d46a14a8b34600f72a", "merged": "2026-01-26", "after_ultra_release": false, "known": [], "repo": "https://github.com/scipp-atlas/pyhs3", "install": { "exclude_newer": "2026-01-27" }, "targets": [ { "target": "pyhs3.generic_parse:parse_expression", "run_id": "bt-pyhs3#154-77bf88", "image": "78fcee79-cf4a-4dca-bb7e-aefefb7d37d3", "backend": "contree", "usd": 0.129, "harvested": 80, "arms": { "hypothesis": { "classes": [], "known": [], "seconds": 32.8, "error": null }, "mutations": { "classes": [], "known": [], "seconds": 25.5, "error": null }, "breakwater": { "classes": [], "known": [], "seconds": 144.0, "error": null }, "llm-plan": { "classes": [], "known": [], "seconds": 20.3, "error": null } }, "findings": [], "perf": { "slower_families": [], "worst_valid_ratio": null, "worst_family": null, "speedup_ratio_small": null, "report": { "run_id": "bt-pyhs3#154-77bf88", "target": "pyhs3.generic_parse:parse_expression", "image": "78fcee79-cf4a-4dca-bb7e-aefefb7d37d3", "backend": "contree", "families": [ { "id": "F1", "kind": "claimed_speedup", "family": "repeated identical expression", "why": "The new code caches the result of `sympy_parser.parse_expr`. When the same expression string is parsed many times the cache turns an O(N) parse into an O(1) dict lookup, so the change is meant to speed up this workload.", "size_meaning": "Number of repeated parses (capped at 8 because the harness only calls the target with ≤8 argument lists).", "make_source": "def make(n):\n \"\"\"Return up to 8 calls all parsing the same expression.\"\"\"\n k = max(1, min(n, 8))\n expr = \"sin(x)+cos(y)*tan(z)\"\n return [[expr] for _ in range(k)]", "example_args": [ [ "sin(x)+cos(y)*tan(z)" ], [ "sin(x)+cos(y)*tan(z)" ] ], "origin": "plan", "screen_ratio": null, "valid_fraction": 1.0, "error": "unmeasured: instruction count not linear in repetitions (caching or state)" }, { "id": "F2", "kind": "claimed_speedup", "family": "The change adds an LRU cache (maxsize 2048) around the SymPy parser. When the same expression string is parsed many times, the cached version returns the previo", "why": "The change adds an LRU cache (maxsize 2048) around the SymPy parser. When the same expression string is parsed many times, the cached version returns the previously computed SymPy object instantly, while the base version re‑parses every call.", "size_meaning": "n = number of repetitions of the same expression", "make_source": "def make(n):\n # a simple but non‑trivial expression that exercises several transformations\n expr = \"sin(x) + cos(y) * 2.5! + lambda z: z**2\"\n return [[expr]] * n", "example_args": [], "origin": "plan", "screen_ratio": null, "valid_fraction": 0.0, "error": "unmeasured: instruction count not linear in repetitions (caching or state)" }, { "id": "F3", "kind": "claimed_speedup", "family": "Repeated identical expression string (cache hit)", "why": "Head version caches the parsed SymPy expression with an LRU cache (maxsize=2048). When the same expression string is parsed many times the cache returns the previously built SymPy object, turning an O(parse) cost into an O(dict‑lookup) cost. Base version re‑parses every call.", "size_meaning": "n controls how many calls are made (1‑8). Larger n → more cache hits.", "make_source": "def make(n):\n \"\"\"Return up to 8 identical argument lists; each list is a single expression string.\"\"\"\n expr = \"sin(x)+cos(y)*tan(z)\"\n count = min(max(n, 1), 8)\n return [[expr] for _ in range(count)]", "example_args": [], "origin": "plan", "screen_ratio": null, "valid_fraction": 1.0, "error": "unmeasured: instruction count not linear in repetitions (caching or state)" }, { "id": "F4", "kind": "suspected_slowdown", "family": "many distinct short expressions", "why": "Each call receives a different short string (e.g. \"x0\", \"x1\", …). The cache is never hit, so the wrapper adds hashing‑and‑dict‑lookup overhead on top of the original parse cost.", "size_meaning": "Number of distinct short expressions parsed (capped at 8).", "make_source": "def make(n):\n \"\"\"Return up to 8 calls each with a different short expression.\"\"\"\n k = max(1, min(n, 8))\n return [[f\"x{i}\"] for i in range(k)]", "example_args": [ [ "x0" ], [ "x1" ] ], "origin": "plan", "screen_ratio": null, "valid_fraction": 1.0, "error": "unmeasured: instruction count not linear in repetitions (caching or state)" }, { "id": "F5", "kind": "suspected_slowdown", "family": "Each call receives a brand‑new expression string. The head version must hash the string, perform a cache lookup (miss), then call the parser; the base version s", "why": "Each call receives a brand‑new expression string. The head version must hash the string, perform a cache lookup (miss), then call the parser; the base version skips the cache entirely. For short strings the extra hash+dict work dominates the parse time, making the head slower.", "size_meaning": "n = number of distinct expressions (each called once)", "make_source": "def make(n):\n # generate distinct short expressions like \"a1\", \"a2\", … using letters and digits\n letters = string.ascii_lowercase\n exprs = []\n for i in range(n):\n ch = letters[i % 26]\n num = i // 26\n exprs.append([f\"{ch}{num}\"])\n return exprs", "example_args": [], "origin": "plan", "screen_ratio": null, "valid_fraction": 1.0, "error": "unmeasured: instruction count not linear in repetitions (caching or state)" }, { "id": "F6", "kind": "suspected_slowdown", "family": "Distinct expression on every call (cache miss)", "why": "Each call receives a previously unseen string, so the LRU cache cannot help. Head still pays the cache‑lookup overhead (hashing the string, dict access) before delegating to SymPy, while base goes straight to the parser. For a workload that never repeats strings the extra indirection is pure overhead.", "size_meaning": "n controls the length/complexity of each distinct expression (larger n → longer strings → more hashing work). Up to 8 calls are generated.", "make_source": "def make(n):\n \"\"\"Generate up to 8 distinct expressions; each grows with n.\"\"\"\n out = []\n for i in range(min(max(n, 1), 8)):\n # a simple unique expression, padded to length ~n\n base = f\"x{i}+{i}*y{i}\"\n if n > 0:\n base += \"+0\" * (n // 2)\n out.append([base])\n return out", "example_args": [], "origin": "plan", "screen_ratio": null, "valid_fraction": 1.0, "error": "unmeasured: instruction count not linear in repetitions (caching or state)" } ], "findings": [], "seconds": { "plan": 11.21, "screen": 1288.27, "full": 0.0, "total": 1299.48 } } } } ] } ], "summary": { "items": 10, "known_verified": 0, "known_candidate": 0, "caught_verified": { "hypothesis": 0, "mutations": 0, "llm-plan": 0, "breakwater": 0 }, "caught_candidate": { "hypothesis": 0, "mutations": 0, "llm-plan": 0, "breakwater": 0 }, "items_with_classes": { "hypothesis": 0, "mutations": 0, "llm-plan": 1, "breakwater": 4 }, "usd": 0.8071, "perf_items_slower": 4, "provider": "tokenfactory" } }