definelist and pragma got tests. Still working on preproc

internal/preproc/definelist_test.go

@@ -0,0 +1,186 @@
+package preproc
+
+import (
+	"testing"
+)
+
+func TestDefineList_AddAndDefined(t *testing.T) {
+	dl := NewDefineList()
+
+	if dl.Defined("FOO") {
+		t.Error("FOO should not be defined initially")
+	}
+
+	dl.Add("FOO", "bar")
+	if !dl.Defined("FOO") {
+		t.Error("FOO should be defined after Add")
+	}
+}
+
+func TestDefineList_Delete(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("FOO", "bar")
+
+	if !dl.Delete("FOO") {
+		t.Error("Delete should return true for existing key")
+	}
+
+	if dl.Defined("FOO") {
+		t.Error("FOO should not be defined after Delete")
+	}
+
+	if dl.Delete("NONEXISTENT") {
+		t.Error("Delete should return false for non-existent key")
+	}
+}
+
+func TestDefineList_ReplaceDefines_Simple(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("FOO", "replacement")
+
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"FOO", "replacement"},
+		{"before FOO after", "before replacement after"},
+		{"FOOFOO", "replacementreplacement"},
+		{"no match here", "no match here"},
+		{"", ""},
+	}
+
+	for _, tt := range tests {
+		got := dl.ReplaceDefines(tt.input)
+		if got != tt.expected {
+			t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected)
+		}
+	}
+}
+
+func TestDefineList_ReplaceDefines_LongestPrefix(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("FOO", "short")
+	dl.Add("FOOBAR", "long")
+
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"FOO", "short"},
+		{"FOOBAR", "long"},       // longest match wins
+		{"FOOBARBAZ", "longBAZ"}, // longest match, rest unchanged
+		{"FOO BAR", "short BAR"}, // space breaks match
+	}
+
+	for _, tt := range tests {
+		got := dl.ReplaceDefines(tt.input)
+		if got != tt.expected {
+			t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected)
+		}
+	}
+}
+
+func TestDefineList_ReplaceDefines_NoRecursion(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("A", "B")
+	dl.Add("B", "C")
+
+	// Should not recursively expand B->C
+	if got := dl.ReplaceDefines("A"); got != "B" {
+		t.Errorf("ReplaceDefines should not recurse: got %q, want %q", got, "B")
+	}
+}
+
+func TestDefineList_ReplaceDefines_CaseSensitive(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("FOO", "bar")
+
+	if got := dl.ReplaceDefines("foo"); got != "foo" {
+		t.Errorf("Should be case-sensitive: got %q, want %q", got, "foo")
+	}
+}
+
+func TestDefineList_ReplaceDefines_UTF8(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("🚀", "rocket")
+	dl.Add("αβ", "alpha-beta")
+
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"🚀", "rocket"},
+		{"test🚀test", "testrockettest"},
+		{"αβγ", "alpha-betaγ"},
+		{"日本語FOO", "日本語FOO"}, // no match
+	}
+
+	for _, tt := range tests {
+		got := dl.ReplaceDefines(tt.input)
+		if got != tt.expected {
+			t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected)
+		}
+	}
+}
+
+func TestDefineList_ReplaceDefines_MultipleReplacements(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("X", "1")
+	dl.Add("Y", "2")
+	dl.Add("Z", "3")
+
+	got := dl.ReplaceDefines("X Y Z X")
+	want := "1 2 3 1"
+	if got != want {
+		t.Errorf("ReplaceDefines = %q, want %q", got, want)
+	}
+}
+
+func TestDefineList_ReplaceDefines_OverlappingPrefixes(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("A", "x")
+	dl.Add("AB", "y")
+	dl.Add("ABC", "z")
+
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"A", "x"},
+		{"AB", "y"},
+		{"ABC", "z"},
+		{"ABCD", "zD"},
+		{"AAA", "xxx"},        // three separate A matches
+		{"ABABC", "yz"},       // AB, then ABC
+		{"A-AB-ABC", "x-y-z"}, // separated by dashes
+	}
+
+	for _, tt := range tests {
+		got := dl.ReplaceDefines(tt.input)
+		if got != tt.expected {
+			t.Errorf("ReplaceDefines(%q) = %q, want %q", tt.input, got, tt.expected)
+		}
+	}
+}
+
+func TestDefineList_ReplaceDefines_EmptyValue(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("REMOVE", "")
+
+	got := dl.ReplaceDefines("before REMOVE after")
+	want := "before  after"
+	if got != want {
+		t.Errorf("ReplaceDefines = %q, want %q", got, want)
+	}
+}
+
+func TestDefineList_ReplaceDefines_UpdateValue(t *testing.T) {
+	dl := NewDefineList()
+	dl.Add("FOO", "first")
+	dl.Add("FOO", "second") // update
+
+	got := dl.ReplaceDefines("FOO")
+	if got != "second" {
+		t.Errorf("After update: got %q, want %q", got, "second")
+	}
+}

internal/preproc/pragma.go

@@ -0,0 +1,44 @@
+package preproc
+
+// PragmaSet is an immutable snapshot of pragma name->value mappings.
+type PragmaSet struct {
+	m map[string]string
+}
+
+// GetPragma returns the value for name or "" if not present.
+func (ps PragmaSet) GetPragma(name string) string {
+	if v, ok := ps.m[name]; ok {
+		return v
+	}
+	return ""
+}
+
+// Pragma manages an immutable stack of PragmaSet snapshots.
+type Pragma struct {
+	pragmaSetStack []PragmaSet
+}
+
+func NewPragma() *Pragma {
+	return &Pragma{
+		pragmaSetStack: []PragmaSet{{m: make(map[string]string)}},
+	}
+}
+
+// AddPragma adds or replaces a pragma by name.
+func (p *Pragma) AddPragma(name, value string) {
+	last := p.pragmaSetStack[len(p.pragmaSetStack)-1].m
+	newMap := make(map[string]string, len(last)+1)
+	for k, v := range last {
+		newMap[k] = v
+	}
+	newMap[name] = value
+	p.pragmaSetStack = append(p.pragmaSetStack, PragmaSet{m: newMap})
+}
+
+func (p *Pragma) GetCurrentPragmaSetIndex() int {
+	return len(p.pragmaSetStack) - 1
+}
+
+func (p *Pragma) GetPragmaSetByIndex(index int) PragmaSet {
+	return p.pragmaSetStack[index] // panics if bad
+}

internal/preproc/pragma_test.go

@@ -0,0 +1,82 @@
+package preproc
+
+import "testing"
+
+func TestNewPragma(t *testing.T) {
+	p := NewPragma()
+	if len(p.pragmaSetStack) != 1 {
+		t.Errorf("expected initial stack length 1, got %d", len(p.pragmaSetStack))
+	}
+	if p.GetCurrentPragmaSetIndex() != 0 {
+		t.Errorf("expected initial index 0, got %d", p.GetCurrentPragmaSetIndex())
+	}
+}
+
+func TestAddPragma(t *testing.T) {
+	p := NewPragma()
+
+	p.AddPragma("foo", "bar")
+	if p.GetCurrentPragmaSetIndex() != 1 {
+		t.Errorf("expected index 1 after add, got %d", p.GetCurrentPragmaSetIndex())
+	}
+
+	ps := p.GetPragmaSetByIndex(1)
+	if ps.GetPragma("foo") != "bar" {
+		t.Errorf("expected 'bar', got '%s'", ps.GetPragma("foo"))
+	}
+}
+
+func TestPragmaImmutability(t *testing.T) {
+	p := NewPragma()
+	p.AddPragma("x", "1")
+	idx1 := p.GetCurrentPragmaSetIndex()
+
+	p.AddPragma("x", "2")
+	idx2 := p.GetCurrentPragmaSetIndex()
+
+	ps1 := p.GetPragmaSetByIndex(idx1)
+	ps2 := p.GetPragmaSetByIndex(idx2)
+
+	if ps1.GetPragma("x") != "1" {
+		t.Errorf("snapshot corrupted: expected '1', got '%s'", ps1.GetPragma("x"))
+	}
+	if ps2.GetPragma("x") != "2" {
+		t.Errorf("expected '2', got '%s'", ps2.GetPragma("x"))
+	}
+}
+
+func TestGetPragmaMissing(t *testing.T) {
+	p := NewPragma()
+	ps := p.GetPragmaSetByIndex(0)
+
+	if ps.GetPragma("missing") != "" {
+		t.Errorf("expected empty string for missing pragma")
+	}
+}
+
+func TestGetPragmaWrongIndex(t *testing.T) {
+	p := NewPragma()
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Errorf("expected panic for index -1")
+		}
+	}()
+
+	p.GetPragmaSetByIndex(-1)
+}
+
+func TestMultiplePragmas(t *testing.T) {
+	p := NewPragma()
+	p.AddPragma("a", "1")
+	p.AddPragma("b", "2")
+
+	ps := p.GetPragmaSetByIndex(p.GetCurrentPragmaSetIndex())
+
+	if ps.GetPragma("a") != "1" {
+		t.Errorf("expected 'a'='1'")
+	}
+	if ps.GetPragma("b") != "2" {
+		t.Errorf("expected 'b'='2'")
+	}
+}

internal/preproc/preproc.go

@@ -1,7 +1,6 @@
 package preproc
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"os"
@@ -15,6 +14,7 @@ type Line struct {
 	Filename       string   // file the line came from (after resolving includes)
 	LineNo         int      // 1-based line number in Filename
 	Tokens         []string // whitespace-split tokens from Text (space or tab; consecutive collapsed)
+	PragmaSetIndex int      // index into Pragma stack for this line
 }
 
 // HaltError is returned when a `#HALT` directive is encountered.
@@ -33,15 +33,17 @@ func PreProcess(rootFilename string) ([]Line, error) {
 
 type preproc struct {
 	defs   *DefineList // from definelist.go
-	pragma map[string]string // #PRAGMA NAME VALUE (stored, not interpreted here)
+	pragma *Pragma     // pragma handler
 	cond   []bool      // conditional stack; a line is active if all are true
+	inAsm  bool        // true when inside ASM/ENDASM block
 }
 
 func newPreproc() *preproc {
 	return &preproc{
 		defs:   NewDefineList(),
-		pragma: make(map[string]string),
+		pragma: NewPragma(),
 		cond:   []bool{},
+		inAsm:  false,
 	}
 }
 
@@ -50,49 +52,99 @@ func (p *preproc) run(root string) ([]Line, error) {
 
 	type frame struct {
 		path  string
-		f    *os.File
+		lines []string // file contents split into lines (no newline)
-		s    *bufio.Scanner
+		idx   int      // next line index (0-based)
-		line int
+		line  int      // last emitted line number (1-based)
 		dir   string
 	}
 
+	// cache of already-read files: fullpath -> []string
+	cache := make(map[string][]string)
+
 	newFrame := func(path string) (*frame, error) {
-		f, err := os.Open(path)
+		abs, err := filepath.Abs(path)
 		if err != nil {
 			return nil, err
 		}
-		sc := bufio.NewScanner(f)
+		if lines, ok := cache[abs]; ok {
-		// Allow long lines (Pascal source and macro expansion can be large)
+			return &frame{path: abs, lines: lines, idx: 0, line: 0, dir: filepath.Dir(abs)}, nil
-		sc.Buffer(make([]byte, 0, 64*1024), 10*1024*1024)
+		}
-		return &frame{path: path, f: f, s: sc, line: 0, dir: filepath.Dir(path)}, nil
+		data, err := os.ReadFile(abs)
+		if err != nil {
+			return nil, err
+		}
+		lines := strings.Split(string(data), "\n")
+		cache[abs] = lines
+		return &frame{path: abs, lines: lines, idx: 0, line: 0, dir: filepath.Dir(abs)}, nil
 	}
 
-	var stack []*frame
+	var frameStack []*frame
 
 	absRoot, _ := filepath.Abs(root)
-	fr, err := newFrame(absRoot)
+	frRoot, err := newFrame(absRoot)
 	if err != nil {
 		return nil, err
 	}
-	stack = append(stack, fr)
+	frameStack = append(frameStack, frRoot)
 
-	for len(stack) > 0 {
+	for len(frameStack) > 0 {
-		fr := stack[len(stack)-1]
+		currFrame := frameStack[len(frameStack)-1]
-		if !fr.s.Scan() {
+
-			if err := fr.s.Err(); err != nil {
+		// if we've exhausted lines in this frame, pop it
-				_ = fr.f.Close()
+		if currFrame.idx >= len(currFrame.lines) {
-				return nil, err
+			frameStack = frameStack[:len(frameStack)-1]
-			}
+			continue
-			_ = fr.f.Close()
+		}
-			stack = stack[:len(stack)-1]
+
+		// advance to next line
+		raw := currFrame.lines[currFrame.idx]
+		currFrame.idx++
+		currFrame.line = currFrame.idx
+
+		includeSource := p.shouldIncludeSource()
+		tokens := strings.Fields(raw)
+
+		// ASM mode handling
+		if !p.inAsm {
+			// Check for ASM entry
+			if includeSource && len(tokens) > 0 && tokens[0] == "ASM" {
+				p.inAsm = true
+				out = append(out, Line{
+					Text:           raw,
+					Filename:       currFrame.path,
+					LineNo:         currFrame.line,
+					Tokens:         []string{},
+					PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(),
+				})
+				continue
+			}
+		} else {
+			// We're in ASM mode
+			// Check for ENDASM
+			if len(tokens) > 0 && tokens[0] == "ENDASM" {
+				p.inAsm = false
+				out = append(out, Line{
+					Text:           raw,
+					Filename:       currFrame.path,
+					LineNo:         currFrame.line,
+					Tokens:         []string{},
+					PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(),
+				})
+				continue
+			}
+			// Otherwise emit line verbatim
+			out = append(out, Line{
+				Text:           raw,
+				Filename:       currFrame.path,
+				LineNo:         currFrame.line,
+				Tokens:         []string{},
+				PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(),
+			})
 			continue
 		}
-		fr.line++
-		raw := fr.s.Text()
 
 		trim := strings.TrimSpace(raw)
 		isDirective := strings.HasPrefix(trim, "#")
-		active := p.isActive()
 
 		if isDirective {
 			parts := strings.Fields(trim)
@@ -101,31 +153,31 @@ func (p *preproc) run(root string) ([]Line, error) {
 			}
 			switch strings.ToUpper(parts[0]) {
 			case "#DEFINE":
-				if active && len(parts) >= 2 {
+				if includeSource && len(parts) >= 2 {
 					name := parts[1]
 					val := ""
 					if len(parts) > 2 {
 						val = strings.Join(parts[2:], " ")
-						val = p.defs.ReplaceDefines(val) // allow nested defines in values
+						val = p.defs.ReplaceDefines(val)
 					}
 					p.defs.Add(name, val)
 				}
 				continue
 			case "#UNDEF":
-				if active && len(parts) >= 2 {
+				if includeSource && len(parts) >= 2 {
 					p.defs.Delete(parts[1])
 				}
 				continue
 			case "#IFDEF":
 				if len(parts) != 2 {
-					return nil, fmt.Errorf("#IFDEF requires exactly one argument at %s:%d", fr.path, fr.line)
+					return nil, fmt.Errorf("#IFDEF requires exactly one argument at %s:%d", currFrame.path, currFrame.line)
 				}
 				p.cond = append(p.cond, p.defs.Defined(parts[1]))
 				continue
 
 			case "#IFNDEF":
 				if len(parts) != 2 {
-					return nil, fmt.Errorf("#IFNDEF requires exactly one argument at %s:%d", fr.path, fr.line)
+					return nil, fmt.Errorf("#IFNDEF requires exactly one argument at %s:%d", currFrame.path, currFrame.line)
 				}
 				p.cond = append(p.cond, !p.defs.Defined(parts[1]))
 				continue
@@ -135,43 +187,43 @@ func (p *preproc) run(root string) ([]Line, error) {
 				}
 				continue
 			case "#INCLUDE":
-				if active {
+				if includeSource {
 					if len(parts) < 2 {
-						return nil, fmt.Errorf("#INCLUDE without path at %s:%d", fr.path, fr.line)
+						return nil, fmt.Errorf("#INCLUDE without path at %s:%d", currFrame.path, currFrame.line)
 					}
 					incPathRaw := parts[1]
-					nextPath, err := resolveInclude(incPathRaw, fr.dir)
+					nextPath, err := resolveInclude(incPathRaw, currFrame.dir)
 					if err != nil {
-						return nil, fmt.Errorf("%s at %s:%d", err, fr.path, fr.line)
+						return nil, fmt.Errorf("%s at %s:%d", err, currFrame.path, currFrame.line)
 					}
 					next, err := newFrame(nextPath)
 					if err != nil {
 						return nil, fmt.Errorf("include open failed %q: %w", nextPath, err)
 					}
-					stack = append(stack, next)
+					frameStack = append(frameStack, next)
 				}
 				continue
 			case "#PRINT":
-				if active {
+				if includeSource {
 					msg := strings.TrimSpace(strings.TrimPrefix(trim, "#PRINT"))
 					msg = p.defs.ReplaceDefines(msg)
 					fmt.Println(msg)
 				}
 				continue
 			case "#HALT":
-				if active {
+				if includeSource {
 					return nil, HaltError{}
 				}
 				continue
 			case "#PRAGMA":
-				if active && len(parts) >= 2 {
+				if includeSource && len(parts) >= 2 {
 					name := strings.ToUpper(parts[1])
 					val := ""
 					if len(parts) > 2 {
 						val = strings.Join(parts[2:], " ")
 						val = p.defs.ReplaceDefines(val)
 					}
-					p.pragma[name] = val
+					p.pragma.AddPragma(name, val)
 				}
 				continue
 			default:
@@ -180,7 +232,7 @@ func (p *preproc) run(root string) ([]Line, error) {
 			}
 		}
 
-		if !active {
+		if !includeSource {
 			continue
 		}
 
@@ -188,22 +240,17 @@ func (p *preproc) run(root string) ([]Line, error) {
 		text := p.defs.ReplaceDefines(raw)
 		out = append(out, Line{
 			Text:           text,
-			Filename: fr.path,
+			Filename:       currFrame.path,
-			LineNo:   fr.line,
+			LineNo:         currFrame.line,
 			Tokens:         strings.Fields(text),
+			PragmaSetIndex: p.pragma.GetCurrentPragmaSetIndex(),
 		})
 	}
 
-	// Check any scanner errors on remaining frames (if any)
-	for _, fr := range stack {
-		if err := fr.s.Err(); err != nil {
-			return nil, err
-		}
-	}
 	return out, nil
 }
 
-func (p *preproc) isActive() bool {
+func (p *preproc) shouldIncludeSource() bool {
 	for _, v := range p.cond {
 		if !v {
 			return false
@@ -237,26 +284,3 @@ func resolveInclude(spec string, curDir string) (string, error) {
 	}
 	return path, nil
 }
-
-// fieldsCollapsed splits on spaces and tabs, ignoring empties (i.e., collapses runs).
-/*
-func fieldsCollapsed(s string) []string {
-	out := make([]string, 0, 8)
-	field := strings.Builder{}
-	flush := func() {
-		if field.Len() > 0 {
-			out = append(out, field.String())
-			field.Reset()
-		}
-	}
-	for _, r := range s {
-		if r == ' ' || r == '\t' {
-			flush()
-		} else {
-			field.WriteRune(r)
-		}
-	}
-	flush()
-	return out
-}
-*/