From f7f247b69ca30b123bccf170f35e434c56ee559b Mon Sep 17 00:00:00 2001 From: Mattias Hansson Date: Sun, 12 Oct 2025 13:54:26 +0200 Subject: [PATCH] definelist and pragma got tests. Still working on preproc --- internal/preproc/definelist_test.go | 186 +++++++++++++++++++++++++++ internal/preproc/pragma.go | 44 +++++++ internal/preproc/pragma_test.go | 82 ++++++++++++ internal/preproc/preproc.go | 190 ++++++++++++++++------------ 4 files changed, 419 insertions(+), 83 deletions(-) create mode 100644 internal/preproc/definelist_test.go create mode 100644 internal/preproc/pragma.go create mode 100644 internal/preproc/pragma_test.go diff --git a/internal/preproc/definelist_test.go b/internal/preproc/definelist_test.go new file mode 100644 index 0000000..cfdb417 --- /dev/null +++ b/internal/preproc/definelist_test.go @@ -0,0 +1,186 @@ +package preproc + +import ( + "testing" +) + +func TestDefineList_AddAndDefined(t *testing.T) { + dl := NewDefineList() + + if dl.Defined("FOO") { + t.Error("FOO should not be defined initially") + } + + dl.Add("FOO", "bar") + if !dl.Defined("FOO") { + t.Error("FOO should be defined after Add") + } +} + +func TestDefineList_Delete(t *testing.T) { + dl := NewDefineList() + dl.Add("FOO", "bar") + + if !dl.Delete("FOO") { + t.Error("Delete should return true for existing key") + } + + if dl.Defined("FOO") { + t.Error("FOO should not be defined after Delete") + } + + if dl.Delete("NONEXISTENT") { + t.Error("Delete should return false for non-existent key") + } +} + +func TestDefineList_ReplaceDefines_Simple(t *testing.T) { + dl := NewDefineList() + dl.Add("FOO", "replacement") + + tests := []struct { + input string + expected string + }{ + {"FOO", "replacement"}, + {"before FOO after", "before replacement after"}, + {"FOOFOO", "replacementreplacement"}, + {"no match here", "no match here"}, + {"", ""}, + } + + for _, tt := range tests { + got := dl.ReplaceDefines(tt.input) + if got != tt.expected { + t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected) + } + } +} + +func TestDefineList_ReplaceDefines_LongestPrefix(t *testing.T) { + dl := NewDefineList() + dl.Add("FOO", "short") + dl.Add("FOOBAR", "long") + + tests := []struct { + input string + expected string + }{ + {"FOO", "short"}, + {"FOOBAR", "long"}, // longest match wins + {"FOOBARBAZ", "longBAZ"}, // longest match, rest unchanged + {"FOO BAR", "short BAR"}, // space breaks match + } + + for _, tt := range tests { + got := dl.ReplaceDefines(tt.input) + if got != tt.expected { + t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected) + } + } +} + +func TestDefineList_ReplaceDefines_NoRecursion(t *testing.T) { + dl := NewDefineList() + dl.Add("A", "B") + dl.Add("B", "C") + + // Should not recursively expand B->C + if got := dl.ReplaceDefines("A"); got != "B" { + t.Errorf("ReplaceDefines should not recurse: got %q, want %q", got, "B") + } +} + +func TestDefineList_ReplaceDefines_CaseSensitive(t *testing.T) { + dl := NewDefineList() + dl.Add("FOO", "bar") + + if got := dl.ReplaceDefines("foo"); got != "foo" { + t.Errorf("Should be case-sensitive: got %q, want %q", got, "foo") + } +} + +func TestDefineList_ReplaceDefines_UTF8(t *testing.T) { + dl := NewDefineList() + dl.Add("🚀", "rocket") + dl.Add("αβ", "alpha-beta") + + tests := []struct { + input string + expected string + }{ + {"🚀", "rocket"}, + {"test🚀test", "testrockettest"}, + {"αβγ", "alpha-betaγ"}, + {"日本語FOO", "日本語FOO"}, // no match + } + + for _, tt := range tests { + got := dl.ReplaceDefines(tt.input) + if got != tt.expected { + t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected) + } + } +} + +func TestDefineList_ReplaceDefines_MultipleReplacements(t *testing.T) { + dl := NewDefineList() + dl.Add("X", "1") + dl.Add("Y", "2") + dl.Add("Z", "3") + + got := dl.ReplaceDefines("X Y Z X") + want := "1 2 3 1" + if got != want { + t.Errorf("ReplaceDefines = %q, want %q", got, want) + } +} + +func TestDefineList_ReplaceDefines_OverlappingPrefixes(t *testing.T) { + dl := NewDefineList() + dl.Add("A", "x") + dl.Add("AB", "y") + dl.Add("ABC", "z") + + tests := []struct { + input string + expected string + }{ + {"A", "x"}, + {"AB", "y"}, + {"ABC", "z"}, + {"ABCD", "zD"}, + {"AAA", "xxx"}, // three separate A matches + {"ABABC", "yz"}, // AB, then ABC + {"A-AB-ABC", "x-y-z"}, // separated by dashes + } + + for _, tt := range tests { + got := dl.ReplaceDefines(tt.input) + if got != tt.expected { + t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected) + } + } +} + +func TestDefineList_ReplaceDefines_EmptyValue(t *testing.T) { + dl := NewDefineList() + dl.Add("REMOVE", "") + + got := dl.ReplaceDefines("before REMOVE after") + want := "before after" + if got != want { + t.Errorf("ReplaceDefines = %q, want %q", got, want) + } +} + +func TestDefineList_ReplaceDefines_UpdateValue(t *testing.T) { + dl := NewDefineList() + dl.Add("FOO", "first") + dl.Add("FOO", "second") // update + + got := dl.ReplaceDefines("FOO") + if got != "second" { + t.Errorf("After update: got %q, want %q", got, "second") + } +} diff --git a/internal/preproc/pragma.go b/internal/preproc/pragma.go new file mode 100644 index 0000000..dcd5966 --- /dev/null +++ b/internal/preproc/pragma.go @@ -0,0 +1,44 @@ +package preproc + +// PragmaSet is an immutable snapshot of pragma name->value mappings. +type PragmaSet struct { + m map[string]string +} + +// GetPragma returns the value for name or "" if not present. +func (ps PragmaSet) GetPragma(name string) string { + if v, ok := ps.m[name]; ok { + return v + } + return "" +} + +// Pragma manages an immutable stack of PragmaSet snapshots. +type Pragma struct { + pragmaSetStack []PragmaSet +} + +func NewPragma() *Pragma { + return &Pragma{ + pragmaSetStack: []PragmaSet{{m: make(map[string]string)}}, + } +} + +// AddPragma adds or replaces a pragma by name. +func (p *Pragma) AddPragma(name, value string) { + last := p.pragmaSetStack[len(p.pragmaSetStack)-1].m + newMap := make(map[string]string, len(last)+1) + for k, v := range last { + newMap[k] = v + } + newMap[name] = value + p.pragmaSetStack = append(p.pragmaSetStack, PragmaSet{m: newMap}) +} + +func (p *Pragma) GetCurrentPragmaSetIndex() int { + return len(p.pragmaSetStack) - 1 +} + +func (p *Pragma) GetPragmaSetByIndex(index int) PragmaSet { + return p.pragmaSetStack[index] // panics if bad +} diff --git a/internal/preproc/pragma_test.go b/internal/preproc/pragma_test.go new file mode 100644 index 0000000..bd9d5a5 --- /dev/null +++ b/internal/preproc/pragma_test.go @@ -0,0 +1,82 @@ +package preproc + +import "testing" + +func TestNewPragma(t *testing.T) { + p := NewPragma() + if len(p.pragmaSetStack) != 1 { + t.Errorf("expected initial stack length 1, got %d", len(p.pragmaSetStack)) + } + if p.GetCurrentPragmaSetIndex() != 0 { + t.Errorf("expected initial index 0, got %d", p.GetCurrentPragmaSetIndex()) + } +} + +func TestAddPragma(t *testing.T) { + p := NewPragma() + + p.AddPragma("foo", "bar") + if p.GetCurrentPragmaSetIndex() != 1 { + t.Errorf("expected index 1 after add, got %d", p.GetCurrentPragmaSetIndex()) + } + + ps := p.GetPragmaSetByIndex(1) + if ps.GetPragma("foo") != "bar" { + t.Errorf("expected 'bar', got '%s'", ps.GetPragma("foo")) + } +} + +func TestPragmaImmutability(t *testing.T) { + p := NewPragma() + p.AddPragma("x", "1") + idx1 := p.GetCurrentPragmaSetIndex() + + p.AddPragma("x", "2") + idx2 := p.GetCurrentPragmaSetIndex() + + ps1 := p.GetPragmaSetByIndex(idx1) + ps2 := p.GetPragmaSetByIndex(idx2) + + if ps1.GetPragma("x") != "1" { + t.Errorf("snapshot corrupted: expected '1', got '%s'", ps1.GetPragma("x")) + } + if ps2.GetPragma("x") != "2" { + t.Errorf("expected '2', got '%s'", ps2.GetPragma("x")) + } +} + +func TestGetPragmaMissing(t *testing.T) { + p := NewPragma() + ps := p.GetPragmaSetByIndex(0) + + if ps.GetPragma("missing") != "" { + t.Errorf("expected empty string for missing pragma") + } +} + +func TestGetPragmaWrongIndex(t *testing.T) { + p := NewPragma() + + defer func() { + if r := recover(); r == nil { + t.Errorf("expected panic for index -1") + } + }() + + p.GetPragmaSetByIndex(-1) +} + +func TestMultiplePragmas(t *testing.T) { + p := NewPragma() + p.AddPragma("a", "1") + p.AddPragma("b", "2") + + ps := p.GetPragmaSetByIndex(p.GetCurrentPragmaSetIndex()) + + if ps.GetPragma("a") != "1" { + t.Errorf("expected 'a'='1'") + } + if ps.GetPragma("b") != "2" { + t.Errorf("expected 'b'='2'") + } +} diff --git a/internal/preproc/preproc.go b/internal/preproc/preproc.go index 91c200d..009ac78 100644 --- a/internal/preproc/preproc.go +++ b/internal/preproc/preproc.go @@ -1,7 +1,6 @@ package preproc import ( - "bufio" "errors" "fmt" "os" @@ -11,10 +10,11 @@ import ( // Line represents one post-processed source line and its provenance. type Line struct { - Text string // post-preprocessor line text (after define replacement) - Filename string // file the line came from (after resolving includes) - LineNo int // 1-based line number in Filename - Tokens []string // whitespace-split tokens from Text (space or tab; consecutive collapsed) + Text string // post-preprocessor line text (after define replacement) + Filename string // file the line came from (after resolving includes) + LineNo int // 1-based line number in Filename + Tokens []string // whitespace-split tokens from Text (space or tab; consecutive collapsed) + PragmaSetIndex int // index into Pragma stack for this line } // HaltError is returned when a `#HALT` directive is encountered. @@ -32,16 +32,18 @@ func PreProcess(rootFilename string) ([]Line, error) { // -------------------- internal -------------------- type preproc struct { - defs *DefineList // from definelist.go - pragma map[string]string // #PRAGMA NAME VALUE (stored, not interpreted here) - cond []bool // conditional stack; a line is active if all are true + defs *DefineList // from definelist.go + pragma *Pragma // pragma handler + cond []bool // conditional stack; a line is active if all are true + inAsm bool // true when inside ASM/ENDASM block } func newPreproc() *preproc { return &preproc{ defs: NewDefineList(), - pragma: make(map[string]string), + pragma: NewPragma(), cond: []bool{}, + inAsm: false, } } @@ -49,50 +51,100 @@ func (p *preproc) run(root string) ([]Line, error) { var out []Line type frame struct { - path string - f *os.File - s *bufio.Scanner - line int - dir string + path string + lines []string // file contents split into lines (no newline) + idx int // next line index (0-based) + line int // last emitted line number (1-based) + dir string } + // cache of already-read files: fullpath -> []string + cache := make(map[string][]string) + newFrame := func(path string) (*frame, error) { - f, err := os.Open(path) + abs, err := filepath.Abs(path) if err != nil { return nil, err } - sc := bufio.NewScanner(f) - // Allow long lines (Pascal source and macro expansion can be large) - sc.Buffer(make([]byte, 0, 64*1024), 10*1024*1024) - return &frame{path: path, f: f, s: sc, line: 0, dir: filepath.Dir(path)}, nil + if lines, ok := cache[abs]; ok { + return &frame{path: abs, lines: lines, idx: 0, line: 0, dir: filepath.Dir(abs)}, nil + } + data, err := os.ReadFile(abs) + if err != nil { + return nil, err + } + lines := strings.Split(string(data), "\n") + cache[abs] = lines + return &frame{path: abs, lines: lines, idx: 0, line: 0, dir: filepath.Dir(abs)}, nil } - var stack []*frame + var frameStack []*frame absRoot, _ := filepath.Abs(root) - fr, err := newFrame(absRoot) + frRoot, err := newFrame(absRoot) if err != nil { return nil, err } - stack = append(stack, fr) + frameStack = append(frameStack, frRoot) - for len(stack) > 0 { - fr := stack[len(stack)-1] - if !fr.s.Scan() { - if err := fr.s.Err(); err != nil { - _ = fr.f.Close() - return nil, err - } - _ = fr.f.Close() - stack = stack[:len(stack)-1] + for len(frameStack) > 0 { + currFrame := frameStack[len(frameStack)-1] + + // if we've exhausted lines in this frame, pop it + if currFrame.idx >= len(currFrame.lines) { + frameStack = frameStack[:len(frameStack)-1] + continue + } + + // advance to next line + raw := currFrame.lines[currFrame.idx] + currFrame.idx++ + currFrame.line = currFrame.idx + + includeSource := p.shouldIncludeSource() + tokens := strings.Fields(raw) + + // ASM mode handling + if !p.inAsm { + // Check for ASM entry + if includeSource && len(tokens) > 0 && tokens[0] == "ASM" { + p.inAsm = true + out = append(out, Line{ + Text: raw, + Filename: currFrame.path, + LineNo: currFrame.line, + Tokens: []string{}, + PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(), + }) + continue + } + } else { + // We're in ASM mode + // Check for ENDASM + if len(tokens) > 0 && tokens[0] == "ENDASM" { + p.inAsm = false + out = append(out, Line{ + Text: raw, + Filename: currFrame.path, + LineNo: currFrame.line, + Tokens: []string{}, + PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(), + }) + continue + } + // Otherwise emit line verbatim + out = append(out, Line{ + Text: raw, + Filename: currFrame.path, + LineNo: currFrame.line, + Tokens: []string{}, + PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(), + }) continue } - fr.line++ - raw := fr.s.Text() trim := strings.TrimSpace(raw) isDirective := strings.HasPrefix(trim, "#") - active := p.isActive() if isDirective { parts := strings.Fields(trim) @@ -101,31 +153,31 @@ func (p *preproc) run(root string) ([]Line, error) { } switch strings.ToUpper(parts[0]) { case "#DEFINE": - if active && len(parts) >= 2 { + if includeSource && len(parts) >= 2 { name := parts[1] val := "" if len(parts) > 2 { val = strings.Join(parts[2:], " ") - val = p.defs.ReplaceDefines(val) // allow nested defines in values + val = p.defs.ReplaceDefines(val) } p.defs.Add(name, val) } continue case "#UNDEF": - if active && len(parts) >= 2 { + if includeSource && len(parts) >= 2 { p.defs.Delete(parts[1]) } continue case "#IFDEF": if len(parts) != 2 { - return nil, fmt.Errorf("#IFDEF requires exactly one argument at %s:%d", fr.path, fr.line) + return nil, fmt.Errorf("#IFDEF requires exactly one argument at %s:%d", currFrame.path, currFrame.line) } p.cond = append(p.cond, p.defs.Defined(parts[1])) continue case "#IFNDEF": if len(parts) != 2 { - return nil, fmt.Errorf("#IFNDEF requires exactly one argument at %s:%d", fr.path, fr.line) + return nil, fmt.Errorf("#IFNDEF requires exactly one argument at %s:%d", currFrame.path, currFrame.line) } p.cond = append(p.cond, !p.defs.Defined(parts[1])) continue @@ -135,43 +187,43 @@ func (p *preproc) run(root string) ([]Line, error) { } continue case "#INCLUDE": - if active { + if includeSource { if len(parts) < 2 { - return nil, fmt.Errorf("#INCLUDE without path at %s:%d", fr.path, fr.line) + return nil, fmt.Errorf("#INCLUDE without path at %s:%d", currFrame.path, currFrame.line) } incPathRaw := parts[1] - nextPath, err := resolveInclude(incPathRaw, fr.dir) + nextPath, err := resolveInclude(incPathRaw, currFrame.dir) if err != nil { - return nil, fmt.Errorf("%s at %s:%d", err, fr.path, fr.line) + return nil, fmt.Errorf("%s at %s:%d", err, currFrame.path, currFrame.line) } next, err := newFrame(nextPath) if err != nil { return nil, fmt.Errorf("include open failed %q: %w", nextPath, err) } - stack = append(stack, next) + frameStack = append(frameStack, next) } continue case "#PRINT": - if active { + if includeSource { msg := strings.TrimSpace(strings.TrimPrefix(trim, "#PRINT")) msg = p.defs.ReplaceDefines(msg) fmt.Println(msg) } continue case "#HALT": - if active { + if includeSource { return nil, HaltError{} } continue case "#PRAGMA": - if active && len(parts) >= 2 { + if includeSource && len(parts) >= 2 { name := strings.ToUpper(parts[1]) val := "" if len(parts) > 2 { val = strings.Join(parts[2:], " ") val = p.defs.ReplaceDefines(val) } - p.pragma[name] = val + p.pragma.AddPragma(name, val) } continue default: @@ -180,30 +232,25 @@ func (p *preproc) run(root string) ([]Line, error) { } } - if !active { + if !includeSource { continue } // Non-directive: expand defines and emit. text := p.defs.ReplaceDefines(raw) out = append(out, Line{ - Text: text, - Filename: fr.path, - LineNo: fr.line, - Tokens: strings.Fields(text), + Text: text, + Filename: currFrame.path, + LineNo: currFrame.line, + Tokens: strings.Fields(text), + PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(), }) } - // Check any scanner errors on remaining frames (if any) - for _, fr := range stack { - if err := fr.s.Err(); err != nil { - return nil, err - } - } return out, nil } -func (p *preproc) isActive() bool { +func (p *preproc) shouldIncludeSource() bool { for _, v := range p.cond { if !v { return false @@ -237,26 +284,3 @@ func resolveInclude(spec string, curDir string) (string, error) { } return path, nil } - -// fieldsCollapsed splits on spaces and tabs, ignoring empties (i.e., collapses runs). -/* -func fieldsCollapsed(s string) []string { - out := make([]string, 0, 8) - field := strings.Builder{} - flush := func() { - if field.Len() > 0 { - out = append(out, field.String()) - field.Reset() - } - } - for _, r := range s { - if r == ' ' || r == '\t' { - flush() - } else { - field.WriteRune(r) - } - } - flush() - return out -} -*/