{ "title": "Weight-Space Programming Reading List", "url": "https://programasweights.com/reading-list/weight-space-programming/", "updated_at": "2026-11-14T07:09:15+00:00", "description": "Program-as-Weights and the research behind programming neural-network weights: trained compilers, generated adapters, formal programs, and LLM reasoning over weights.", "groups": [ { "id": "neural-programs", "title": "Reusable neural programs", "description": "Build a function once, save its program, and execute it on new inputs. PAW defines the compiler–interpreter interface; Compile by Training adds a compilation method." }, { "id": "learned-weight-generation", "title": "Trained weight generators", "description": "Train one network to generate another network’s parameters from task descriptions, examples, or repository contents." }, { "id": "formal-compilers", "title": "Formal program compilation", "description": "Translate a program written in a formal language into a network with a known computation." }, { "id": "direct-weight-programming", "title": "Reasoning over numerical weights", "description": "Use a pretrained LLM to inspect parameters, write networks, or edit existing weights through reasoning and tools." } ], "reading_path": [ { "id": "program-as-weights", "text": "Start with the programming paradigm: a large compiler builds a saved program for a small interpreter." }, { "id": "hypernetworks", "text": "Read the foundation for using one neural network to generate another network’s weights." }, { "id": "text-to-lora", "text": "See how a trained hypernetwork generates adapters from task descriptions." }, { "id": "sparks-weight-space-programming", "text": "Compare trained compilation with a general-purpose LLM reading and writing numerical weights." }, { "id": "exploring-weight-space-programming", "text": "Follow reasoning over weights into a study of repeated searches and cross-task transfer." } ], "records": [ { "id": "hypernetworks", "title": "HyperNetworks", "authors": [ "David Ha", "Andrew Dai", "Quoc V. Le" ], "first_public": "2016-09-27", "venue": "ICLR 2017", "paper_url": "https://arxiv.org/abs/1609.09106", "summary": "Uses one neural network to generate the weights of another.", "conditioning": "Layer embeddings or the current recurrent state.", "program_form": "Weights for the target network.", "execution": "The target network runs with the generated weights.", "relationship": "HyperNetworks supplies a mechanism for generating weights. PAW trains a compiler to use generated parameters as reusable programs.", "construction": "A trained network generates the parameters of another network.", "training": "The weight generator is trained jointly with the target network.", "group": "learned-weight-generation", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/1609.09106", "kind": "paper", "paper_id": "1609.09106", "details_url": "https://programasweights.com/reading-list/papers/1609.09106/", "bibtex_url": "https://programasweights.com/reading-list/papers/1609.09106/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/1609.09106/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#hypernetworks" }, { "id": "hypter", "title": "Learning to Generate Task-Specific Adapters from Task Description", "authors": [ "Qinyuan Ye", "Xiang Ren" ], "first_public": "2021-01-02", "venue": "ACL-IJCNLP 2021", "paper_url": "https://arxiv.org/abs/2101.00420", "summary": "Introduces Hypter, which generates adapters from natural-language task descriptions.", "conditioning": "A natural-language task description.", "program_form": "Bottleneck adapters inserted into a text-to-text model.", "execution": "BART performs the task with the generated adapters.", "relationship": "An early description-to-adapter method cited by PAW. PAW develops the programming interface for saving and calling the resulting neural functions.", "construction": "A description encoder and hypernetwork generate adapters for a text-to-text model.", "training": "The generator learns from tasks with descriptions and labeled examples.", "group": "learned-weight-generation", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2101.00420", "kind": "paper", "paper_id": "2101.00420", "details_url": "https://programasweights.com/reading-list/papers/2101.00420/", "bibtex_url": "https://programasweights.com/reading-list/papers/2101.00420/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/2101.00420/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#hypter" }, { "id": "hypertuning", "title": "HyperTuning: Toward Adapting Large Language Models without Back-propagation", "authors": [ "Jason Phang", "Yi Mao", "Pengcheng He", "Weizhu Chen" ], "first_public": "2022-11-22", "venue": "ICML 2023", "paper_url": "https://arxiv.org/abs/2211.12485", "summary": "Produces task-specific parameters for a frozen language model from a few demonstrations.", "conditioning": "A few examples of the task.", "program_form": "Generated prefix or LoRA parameters.", "execution": "A frozen T5 model runs with the generated parameters.", "relationship": "PAW cites HyperTuning as prior work on generating adaptation parameters from examples.", "construction": "HyperT5 generates prefix or LoRA parameters from task demonstrations.", "training": "Hyperpretraining and multitask training prepare the generator; the paper also studies further task-specific tuning.", "group": "learned-weight-generation", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2211.12485", "kind": "paper", "paper_id": "2211.12485", "details_url": "https://programasweights.com/reading-list/papers/2211.12485/", "bibtex_url": "https://programasweights.com/reading-list/papers/2211.12485/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/2211.12485/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#hypertuning" }, { "id": "hint", "title": "HINT: Hypernetwork Instruction Tuning for Efficient Zero- & Few-Shot Generalisation", "authors": [ "Hamish Ivison", "Akshita Bhagia", "Yizhong Wang", "Hannaneh Hajishirzi", "Matthew Peters" ], "first_public": "2022-12-20", "venue": "ACL 2023", "paper_url": "https://arxiv.org/abs/2212.10315", "summary": "Turns instructions and examples into reusable prefixes, adapters, and encoded instruction representations.", "conditioning": "Task instructions and optional examples.", "program_form": "Prefixes, adapters, and encoded instruction representations.", "execution": "An encoder-decoder model uses the generated modules and attends to cached instruction representations.", "relationship": "HINT reuses generated modules and an encoded instruction. PAW reuses a program containing text and adaptation parameters.", "construction": "An instruction-conditioned hypernetwork generates modules, and instruction fusion caches the instruction representation.", "training": "Instruction tuning trains the generator and model to generalize across tasks.", "group": "learned-weight-generation", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2212.10315", "kind": "paper", "paper_id": "2212.10315", "details_url": "https://programasweights.com/reading-list/papers/2212.10315/", "bibtex_url": "https://programasweights.com/reading-list/papers/2212.10315/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/2212.10315/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#hint" }, { "id": "tracr", "title": "Tracr: Compiled Transformers as a Laboratory for Interpretability", "authors": [ "David Lindner", "János Kramár", "Sebastian Farquhar", "Matthew Rahtz", "Thomas McGrath", "Vladimir Mikulik" ], "first_public": "2023-01-12", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2301.05062", "kind": "paper", "venue": "NeurIPS 2023", "paper_url": "https://arxiv.org/abs/2301.05062", "full_text": "https://arxiv.org/html/2301.05062", "code_url": "https://github.com/google-deepmind/tracr", "method_source": "https://arxiv.org/html/2301.05062#S3", "summary": "Compiles RASP programs into transformer weights with known computational structure.", "conditioning": "A RASP program, input vocabulary, and maximum sequence length.", "program_form": "Transformer weights assembled from attention and MLP components.", "construction": "A symbolic compiler translates each RASP operation into network components and assembles their weights.", "training": "Compilation constructs the weights without model training. A separate experiment trains a projection to compress the compiled model.", "execution": "The compiled transformer executes the specified sequence computation.", "relationship": "Tracr compiles a formal program. PAW learns to compile natural-language specifications of fuzzy functions into programs for a shared interpreter.", "group": "formal-compilers", "bibtex_url": "https://programasweights.com/reading-list/weight-space-programming/citations/tracr.bib", "csl_url": "https://programasweights.com/reading-list/weight-space-programming/citations/tracr.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#tracr" }, { "id": "text-to-lora", "title": "Text-to-LoRA: Instant Transformer Adaption", "authors": [ "Rujikorn Charakorn", "Edoardo Cetin", "Yujin Tang", "Robert Tjarko Lange" ], "first_public": "2025-06-06", "venue": "ICML 2025", "paper_url": "https://arxiv.org/abs/2506.06105", "summary": "Uses a hypernetwork to generate LoRA adapters from task descriptions.", "conditioning": "A natural-language task description.", "program_form": "Generated LoRA matrices.", "execution": "A base transformer performs the task with the generated LoRA.", "relationship": "PAW cites Text-to-LoRA as prior work on generating adapters. PAW develops fuzzy-function programming, FuzzyBench, and an interface for reusable neural programs.", "construction": "A hypernetwork maps the task description to LoRA matrices.", "training": "The paper trains generators by reconstructing adapters and through supervised task learning.", "group": "learned-weight-generation", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2506.06105", "kind": "paper", "paper_id": "2506.06105", "details_url": "https://programasweights.com/reading-list/papers/2506.06105/", "bibtex_url": "https://programasweights.com/reading-list/papers/2506.06105/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/2506.06105/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#text-to-lora" }, { "id": "code-to-lora", "title": "Code2LoRA: Hypernetwork-Generated Adapters for Code Language Models under Software Evolution", "authors": [ "Liliana Hotsko", "Yinxi Li", "Yuntian Deng", "Pengyu Nie" ], "first_public": "2026-06-04", "venue": "arXiv", "paper_url": "https://arxiv.org/abs/2606.06492", "full_text": "https://arxiv.org/html/2606.06492v2", "method_source": "https://arxiv.org/html/2606.06492v2#S3", "summary": "Trains a hypernetwork to generate repository-specific LoRA adapters from source code and update them as commits arrive.", "conditioning": "A repository snapshot; the evolution variant also receives successive commit diffs.", "program_form": "Generated LoRA matrices for a frozen code language model.", "execution": "The code model uses the generated adapter without adding repository context tokens to each request.", "relationship": "Code2LoRA and PAW train models to generate reusable weights. Code2LoRA puts repository knowledge into adapters; PAW compiles function specifications into neural programs with a compiler–interpreter interface.", "construction": "Static maps an encoded repository snapshot to LoRA weights. Evo updates a GRU state with each commit diff and generates the weights from that state.", "training": "The hypernetwork learns across repositories through the frozen code model’s assertion-completion loss. New repositories receive generated adapters without per-repository fine-tuning.", "group": "learned-weight-generation", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2606.06492", "kind": "paper", "paper_id": "2606.06492", "details_url": "https://programasweights.com/reading-list/papers/2606.06492/", "bibtex_url": "https://programasweights.com/reading-list/papers/2606.06492/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/2606.06492/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#code-to-lora" }, { "id": "program-as-weights", "title": "Program-as-Weights: A Programming Paradigm for Fuzzy Functions", "authors": [ "Wentao Zhang", "Liliana Hotsko", "Woojeong Kim", "Pengyu Nie", "Stuart Shieber", "Yuntian Deng" ], "first_public": "2026-07-02", "venue": "NeurIPS 2026", "paper_url": "https://arxiv.org/abs/2607.02512", "code_url": "https://github.com/programasweights", "summary": "Proposes fuzzy-function programming: compile a natural-language function specification into a neural program, then reuse it locally.", "conditioning": "A function specification, optionally with examples. The image extension also accepts visual information.", "program_form": "A discrete pseudo-program and a compiler-generated parameter-efficient module.", "execution": "A frozen lightweight interpreter loads the program and runs new inputs locally.", "relationship": "PAW makes weight-space programming a software interface: define a fuzzy function, compile it, save the neural program, and call it locally. The large model is the compiler; the small model is the interpreter.", "construction": "A large trained compiler produces a neural program from the function specification. Its fast compiler generates LoRA weights in one forward pass.", "training": "The compiler learns to generate programs before deployment. New functions compile without per-function gradient descent.", "group": "neural-programs", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2607.02512", "kind": "paper", "paper_id": "2607.02512", "details_url": "https://programasweights.com/reading-list/papers/2607.02512/", "bibtex_url": "https://programasweights.com/reading-list/papers/2607.02512/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/2607.02512/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#program-as-weights" }, { "id": "compile-by-training", "title": "Compile by Training: Turning Natural-Language Specifications into Local Neural Functions", "authors": [ "Yuntian Deng", "Pengyu Nie", "Stuart Shieber" ], "first_public": "2026-09-03", "venue": "EMNLP 2026 System Demonstrations", "paper_url": "https://aclanthology.org/2026.emnlp-demos.78/", "code_url": "https://github.com/programasweights/compile-by-training", "summary": "Compiles a function description into a reusable local program by generating training examples and optimizing an adapter.", "conditioning": "The specification guides teacher synthesis. PAW’s fast compiler supplies the initial adapter and runtime prompt.", "program_form": "A LoRA adapter, runtime prompt, specification, and interpreter metadata in a PAW program.", "execution": "A frozen Qwen3-0.6B interpreter runs the saved function locally without further teacher calls.", "relationship": "Extends PAW with training at compile time while keeping its program format and runtime interface.", "construction": "Teacher models generate examples from the specification. Gradient descent refines the adapter initialized by PAW’s fast compiler.", "training": "Training happens at function compilation; the saved program runs without the teachers.", "group": "neural-programs", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2609.04199", "kind": "paper", "paper_id": "2609.04199", "details_url": "https://programasweights.com/reading-list/papers/2609.04199/", "bibtex_url": "https://programasweights.com/reading-list/papers/2609.04199/citation.bib", "csl_url": "https://programasweights.com/reading-list/papers/2609.04199/citation.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#compile-by-training" }, { "id": "sparks-weight-space-programming", "title": "Sparks of Weight-Space Programming: LLMs can natively read & write neural network weights", "authors": [ "Rikiya Takehi", "Jayden Teoh", "Alex Zhang", "Omar Khattab" ], "first_public": "2026-11-02", "date_basis": "Article publication date", "date_source": "https://rikiyat.github.io/blog/weight-space-programming/", "kind": "article", "venue": "Research article", "paper_url": "https://rikiyat.github.io/blog/weight-space-programming/", "full_text": "https://rikiyat.github.io/blog/weight-space-programming/", "code_url": "https://github.com/RikiyaT/lm_understand-logs", "method_source": "https://rikiyat.github.io/blog/weight-space-programming/", "summary": "Uses a general-purpose LLM to read neural-network weights, construct networks, and edit an existing model through numerical reasoning and tools.", "conditioning": "A natural-language objective, network weights or an empty network, and task-dependent tool access.", "program_form": "New network weights or an edit to an existing checkpoint.", "construction": "The LLM reasons about the computation and writes the numerical parameters. Larger experiments use a Python environment and forward-pass feedback.", "training": "The reported weight-construction and editing procedures prohibit gradient descent and training loops.", "execution": "The constructed or edited network runs on new inputs.", "relationship": "PAW generates reusable neural programs with a trained compiler. Sparks uses a general-purpose LLM to inspect and write numerical weights directly.", "group": "direct-weight-programming", "bibtex_url": "https://programasweights.com/reading-list/weight-space-programming/citations/sparks-weight-space-programming.bib", "csl_url": "https://programasweights.com/reading-list/weight-space-programming/citations/sparks-weight-space-programming.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#sparks-weight-space-programming" }, { "id": "weight-space-programming-simple-tasks", "title": "Weight-Space Programming from scratch on simple tasks", "authors": [ "Chandan Singh" ], "first_public": "2026-11-02", "date_basis": "Article publication date", "date_source": "https://csinva.io/weight-space-programming-autoresearch/analyze/toy_examples/report.html", "kind": "article", "venue": "Research article", "paper_url": "https://csinva.io/weight-space-programming-autoresearch/analyze/toy_examples/report.html", "full_text": "https://csinva.io/weight-space-programming-autoresearch/analyze/toy_examples/report.html", "code_url": "https://github.com/csinva/weight-space-programming-autoresearch", "method_source": "https://csinva.io/weight-space-programming-autoresearch/analyze/toy_examples/report.html", "summary": "An LLM coding agent writes transformer weights for tasks such as addition, sorting, and parity, then revises the programs using evaluation feedback.", "conditioning": "A task definition and a transformer implementation with an empty weight-writing function.", "program_form": "A transformer architecture and its explicitly assigned weights.", "construction": "The agent edits the architecture and parameter-setting code, runs the resulting model, and simplifies successful programs.", "training": "The algorithmic-task procedure uses closed-form parameter assignment. The report identifies fitting and evaluation leakage in its language-task runs.", "execution": "The written transformer generates its answer token by token.", "relationship": "Both this work and PAW produce neural programs that run after construction. Here an LLM writes the implementation through iterative code edits; PAW uses a trained compiler and a shared interpreter.", "group": "direct-weight-programming", "bibtex_url": "https://programasweights.com/reading-list/weight-space-programming/citations/weight-space-programming-simple-tasks.bib", "csl_url": "https://programasweights.com/reading-list/weight-space-programming/citations/weight-space-programming-simple-tasks.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#weight-space-programming-simple-tasks" }, { "id": "exploring-weight-space-programming", "title": "Exploring Weight-Space Programming: Capabilities, Generalization, and Limitations", "authors": [ "Junru Hu", "Cheng Tang", "Wei Li", "Hongze Zhu", "Chun-Mei Feng", "Qi Zhang", "Yirong Chen", "Ming Hu" ], "first_public": "2026-11-06", "date_basis": "First arXiv submission", "date_source": "https://arxiv.org/abs/2611.09382", "kind": "paper", "venue": "arXiv", "paper_url": "https://arxiv.org/abs/2611.09382", "full_text": "https://arxiv.org/html/2611.09382v1", "code_url": "https://github.com/WSP-Scope/WSP-Scope-Project", "method_source": "https://arxiv.org/html/2611.09382v1#S2", "summary": "Studies direct weight editing across 16 searches, seven benchmark families, and two model scales, measuring held-out gains and transfer between tasks.", "conditioning": "A pretrained Qwen3.5 checkpoint, task data, and generation or teacher-forcing feedback.", "program_form": "A modified pretrained checkpoint.", "construction": "A reasoning agent inspects parameters, writes weight updates, and selects a checkpoint using development feedback.", "training": "The editing search uses no gradient-based optimization. Each search has a budget of 50 successful forward-tool requests.", "execution": "The selected checkpoint answers held-out questions without the editing agent.", "relationship": "This study evaluates reasoning-driven edits to an existing model. PAW compiles a function definition into a separate reusable program for its interpreter.", "group": "direct-weight-programming", "bibtex_url": "https://programasweights.com/reading-list/weight-space-programming/citations/exploring-weight-space-programming.bib", "csl_url": "https://programasweights.com/reading-list/weight-space-programming/citations/exploring-weight-space-programming.csl.json", "record_url": "https://programasweights.com/reading-list/weight-space-programming/#exploring-weight-space-programming" } ] }