Preprocessor first version. Unit test covered. Nothing tested yet with real code.

internal/preproc/filereader.go

@@ -9,7 +9,7 @@ import (
 )
 
 type FileReader interface {
-	ReadLines(includeSpec string, currentDir string) ([]string, error)
+	ReadLines(includeSpec string, currentDir string) ([]string, string, error)
 }
 
 type DiskFileReader struct {
@@ -24,29 +24,29 @@ func NewDiskFileReader() *DiskFileReader {
 	}
 }
 
-func (d *DiskFileReader) ReadLines(includeSpec string, currentDir string) ([]string, error) {
+func (d *DiskFileReader) ReadLines(includeSpec string, currentDir string) ([]string, string, error) {
 	path, err := d.resolvePath(includeSpec, currentDir)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	abs, err := filepath.Abs(path)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	if lines, ok := d.cache[abs]; ok {
-		return lines, nil
+		return lines, abs, nil
 	}
 
 	data, err := os.ReadFile(abs)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	lines := strings.Split(string(data), "\n")
 	d.cache[abs] = lines
-	return lines, nil
+	return lines, abs, nil
 }
 
 func (d *DiskFileReader) resolvePath(spec string, curDir string) (string, error) {
@@ -86,10 +86,10 @@ func NewMockFileReader(files map[string][]string) *MockFileReader {
 	}
 }
 
-func (m *MockFileReader) ReadLines(includeSpec string, _ string) ([]string, error) {
+func (m *MockFileReader) ReadLines(includeSpec string, _ string) ([]string, string, error) {
 	lines, ok := m.files[includeSpec]
 	if !ok {
-		return nil, fmt.Errorf("open %s: no such file or directory", includeSpec)
+		return nil, "", fmt.Errorf("open %s: no such file or directory", includeSpec)
 	}
-	return lines, nil
+	return lines, includeSpec, nil
 }

internal/preproc/filereader_test.go

@@ -40,7 +40,7 @@ func TestDiskFileReader_ReadLines_LibraryIncludes(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			lines, err := reader.ReadLines(tt.includeSpec, "")
+			lines, _, err := reader.ReadLines(tt.includeSpec, "")
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ReadLines() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -61,7 +61,7 @@ func TestDiskFileReader_ReadLines_LibraryWithoutEnv(t *testing.T) {
 	// Now create reader
 	reader := NewDiskFileReader()
 
-	_, err := reader.ReadLines("<test_lib.c65>", "")
+	_, _, err := reader.ReadLines("<test_lib.c65>", "")
 	if err == nil {
 		t.Error("expected error when C65LIBPATH not set")
 	}
@@ -112,7 +112,7 @@ func TestDiskFileReader_ReadLines_RelativeIncludes(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			lines, err := reader.ReadLines(tt.includeSpec, tt.currentDir)
+			lines, _, err := reader.ReadLines(tt.includeSpec, tt.currentDir)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ReadLines() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -130,13 +130,13 @@ func TestDiskFileReader_ReadLines_Cache(t *testing.T) {
 	appDir, _ := filepath.Abs("filereader_mocks/app")
 
 	// First read
-	lines1, err := reader.ReadLines("test_app.c65", appDir)
+	lines1, _, err := reader.ReadLines("test_app.c65", appDir)
 	if err != nil {
 		t.Fatalf("first read failed: %v", err)
 	}
 
 	// Second read (should hit cache)
-	lines2, err := reader.ReadLines("test_app.c65", appDir)
+	lines2, _, err := reader.ReadLines("test_app.c65", appDir)
 	if err != nil {
 		t.Fatalf("second read failed: %v", err)
 	}
@@ -157,7 +157,7 @@ func TestDiskFileReader_ReadLines_EmptyFile(t *testing.T) {
 
 	appDir, _ := filepath.Abs("filereader_mocks/app")
 
-	lines, err := reader.ReadLines("empty.c65", appDir)
+	lines, _, err := reader.ReadLines("empty.c65", appDir)
 	if err != nil {
 		t.Fatalf("failed to read empty file: %v", err)
 	}
@@ -171,7 +171,7 @@ func TestDiskFileReader_ReadLines_EmptyFile(t *testing.T) {
 func TestDiskFileReader_ReadLines_EmptySpec(t *testing.T) {
 	reader := NewDiskFileReader()
 
-	_, err := reader.ReadLines("", "")
+	_, _, err := reader.ReadLines("", "")
 	if err == nil {
 		t.Error("expected error for empty include spec")
 	}
@@ -185,7 +185,7 @@ func TestMockFileReader_ReadLines(t *testing.T) {
 
 	reader := NewMockFileReader(files)
 
-	lines, err := reader.ReadLines("test.c65", "")
+	lines, _, err := reader.ReadLines("test.c65", "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -193,7 +193,7 @@ func TestMockFileReader_ReadLines(t *testing.T) {
 		t.Errorf("expected 2 lines, got %d", len(lines))
 	}
 
-	_, err = reader.ReadLines("notfound", "")
+	_, _, err = reader.ReadLines("notfound", "")
 	if err == nil {
 		t.Error("expected error for missing file")
 	}
@@ -206,7 +206,7 @@ func TestMockFileReader_LibraryInclude(t *testing.T) {
 
 	reader := NewMockFileReader(files)
 
-	lines, err := reader.ReadLines("<test_lib.c65>", "")
+	lines, _, err := reader.ReadLines("<test_lib.c65>", "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -222,7 +222,7 @@ func TestMockFileReader_NormalInclude(t *testing.T) {
 
 	reader := NewMockFileReader(files)
 
-	lines, err := reader.ReadLines("test_app.c65", "some/dir")
+	lines, _, err := reader.ReadLines("test_app.c65", "some/dir")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -238,7 +238,7 @@ func TestMockFileReader_MissingFile(t *testing.T) {
 
 	reader := NewMockFileReader(files)
 
-	_, err := reader.ReadLines("notfound.c65", "")
+	_, _, err := reader.ReadLines("notfound.c65", "")
 	if err == nil {
 		t.Error("expected error for missing file")
 	}

internal/preproc/preproc.go

@@ -1,9 +1,7 @@
 package preproc
 
 import (
-	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 )
@@ -24,8 +22,14 @@ func (HaltError) Error() string { return "preprocessor HALT" }
 
 // PreProcess processes the given root source file and returns the flattened, define-expanded
 // lines. Directive lines are not emitted.
-func PreProcess(rootFilename string) ([]Line, error) {
+func PreProcess(rootFilename string, reader ...FileReader) ([]Line, error) {
-	pp := newPreproc()
+	var r FileReader
+	if len(reader) > 0 && reader[0] != nil {
+		r = reader[0]
+	} else {
+		r = NewDiskFileReader()
+	}
+	pp := newPreproc(r)
 	return pp.run(rootFilename)
 }
 
@@ -36,14 +40,16 @@ type preproc struct {
 	pragma *Pragma     // pragma handler
 	cond   []bool      // conditional stack; a line is active if all are true
 	inAsm  bool        // true when inside ASM/ENDASM block
+	reader FileReader  // file reader abstraction
 }
 
-func newPreproc() *preproc {
+func newPreproc(reader FileReader) *preproc {
 	return &preproc{
 		defs:   NewDefineList(),
 		pragma: NewPragma(),
 		cond:   []bool{},
 		inAsm:  false,
+		reader: reader,
 	}
 }
 
@@ -58,30 +64,23 @@ func (p *preproc) run(root string) ([]Line, error) {
 		dir   string
 	}
 
-	// cache of already-read files: fullpath -> []string
+	newFrame := func(spec string, curDir string) (*frame, error) {
-	cache := make(map[string][]string)
+		lines, absPath, err := p.reader.ReadLines(spec, curDir)
-
-	newFrame := func(path string) (*frame, error) {
-		abs, err := filepath.Abs(path)
 		if err != nil {
 			return nil, err
 		}
-		if lines, ok := cache[abs]; ok {
+		return &frame{
-			return &frame{path: abs, lines: lines, idx: 0, line: 0, dir: filepath.Dir(abs)}, nil
+			path:  absPath,
-		}
+			lines: lines,
-		data, err := os.ReadFile(abs)
+			idx:   0,
-		if err != nil {
+			line:  0,
-			return nil, err
+			dir:   filepath.Dir(absPath),
-		}
+		}, nil
-		lines := strings.Split(string(data), "\n")
-		cache[abs] = lines
-		return &frame{path: abs, lines: lines, idx: 0, line: 0, dir: filepath.Dir(abs)}, nil
 	}
 
 	var frameStack []*frame
 
-	absRoot, _ := filepath.Abs(root)
+	frRoot, err := newFrame(root, "")
-	frRoot, err := newFrame(absRoot)
 	if err != nil {
 		return nil, err
 	}
@@ -157,9 +156,15 @@ func (p *preproc) run(root string) ([]Line, error) {
 					name := parts[1]
 					val := ""
 					if len(parts) > 2 {
-						val = strings.Join(parts[2:], " ")
+						start := 2
+						if parts[2] == "=" {
+							start = 3
+						}
+						if start < len(parts) {
+							val = strings.Join(parts[start:], " ")
 							val = p.defs.ReplaceDefines(val)
 						}
+					}
 					p.defs.Add(name, val)
 				}
 				continue
@@ -192,13 +197,9 @@ func (p *preproc) run(root string) ([]Line, error) {
 						return nil, fmt.Errorf("#INCLUDE without path at %s:%d", currFrame.path, currFrame.line)
 					}
 					incPathRaw := parts[1]
-					nextPath, err := resolveInclude(incPathRaw, currFrame.dir)
+					next, err := newFrame(incPathRaw, currFrame.dir)
 					if err != nil {
-						return nil, fmt.Errorf("%s at %s:%d", err, currFrame.path, currFrame.line)
+						return nil, fmt.Errorf("include failed at %s:%d: %w", currFrame.path, currFrame.line, err)
-					}
-					next, err := newFrame(nextPath)
-					if err != nil {
-						return nil, fmt.Errorf("include open failed %q: %w", nextPath, err)
 					}
 					frameStack = append(frameStack, next)
 				}
@@ -212,7 +213,7 @@ func (p *preproc) run(root string) ([]Line, error) {
 				continue
 			case "#HALT":
 				if includeSource {
-					return nil, HaltError{}
+					return out, HaltError{}
 				}
 				continue
 			case "#PRAGMA":
@@ -247,6 +248,10 @@ func (p *preproc) run(root string) ([]Line, error) {
 		})
 	}
 
+	if len(p.cond) > 0 {
+		return nil, fmt.Errorf("unbalanced conditionals: %d unclosed #IFDEF/#IFNDEF directives", len(p.cond))
+	}
+
 	return out, nil
 }
 
@@ -258,29 +263,3 @@ func (p *preproc) shouldIncludeSource() bool {
 	}
 	return true
 }
-
-func resolveInclude(spec string, curDir string) (string, error) {
-	if len(spec) == 0 {
-		return "", errors.New("empty include path")
-	}
-	// <file> -> library include via C65LIBPATH
-	if strings.HasPrefix(spec, "<") && strings.HasSuffix(spec, ">") {
-		base := strings.TrimSpace(spec[1 : len(spec)-1])
-		lib := os.Getenv("C65LIBPATH")
-		if lib == "" {
-			return "", errors.New("C65LIBPATH not set for angle-bracket include")
-		}
-		path := filepath.Join(lib, base)
-		if _, err := os.Stat(path); err != nil {
-			return "", fmt.Errorf("library include not found: %s", path)
-		}
-		return path, nil
-	}
-	// quoted or bare -> relative to current file dir
-	base := strings.Trim(spec, `"'`)
-	path := filepath.Join(curDir, base)
-	if _, err := os.Stat(path); err != nil {
-		return "", fmt.Errorf("include not found: %s", path)
-	}
-	return path, nil
-}

internal/preproc/preproc_test.go

@@ -0,0 +1,725 @@
+package preproc
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestPreProcess_BasicDefine(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#DEFINE FOO = 42",
+			"LDA #FOO",
+			"STA $D020",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 2 {
+		t.Fatalf("expected 2 lines, got %d", len(lines))
+	}
+
+	if lines[0].Text != "LDA #42" {
+		t.Errorf("expected 'LDA #42', got %q", lines[0].Text)
+	}
+}
+
+func TestPreProcess_DefineExpansion(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#DEFINE BASE = $D000",
+			"#DEFINE OFFSET = 32",
+			"#DEFINE ADDR = BASE+OFFSET",
+			"STA ADDR",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 1 {
+		t.Fatalf("expected 1 line, got %d", len(lines))
+	}
+
+	if lines[0].Text != "STA $D000+32" {
+		t.Errorf("expected 'STA $D000+32', got %q", lines[0].Text)
+	}
+}
+
+func TestPreProcess_IncludeGuard(t *testing.T) {
+	files := map[string][]string{
+		"lib.c65": {
+			"#IFNDEF __LIB",
+			"#DEFINE __LIB = 1",
+			"LABEL lib_init",
+			"#IFEND",
+		},
+		"main.c65": {
+			"#INCLUDE lib.c65",
+			"#INCLUDE lib.c65",
+			"GOTO main",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("main.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	// First include emits LABEL, second include is blocked by guard
+	if len(lines) != 2 {
+		t.Fatalf("expected 2 lines, got %d", len(lines))
+	}
+
+	if lines[0].Text != "LABEL lib_init" {
+		t.Errorf("expected 'LABEL lib_init', got %q", lines[0].Text)
+	}
+	if lines[1].Text != "GOTO main" {
+		t.Errorf("expected 'GOTO main', got %q", lines[1].Text)
+	}
+}
+
+func TestPreProcess_ASMBlock(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#DEFINE FOO = 42",
+			"ASM",
+			"  lda #FOO",
+			"  sta $d020",
+			"ENDASM",
+			"LDA #FOO",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 5 {
+		t.Fatalf("expected 5 lines, got %d", len(lines))
+	}
+
+	// ASM content should NOT be processed
+	if lines[1].Text != "  lda #FOO" {
+		t.Errorf("expected '  lda #FOO', got %q", lines[1].Text)
+	}
+
+	// After ENDASM, defines work again
+	if lines[4].Text != "LDA #42" {
+		t.Errorf("expected 'LDA #42', got %q", lines[4].Text)
+	}
+}
+
+func TestPreProcess_NestedConditionals(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#DEFINE OUTER",
+			"#IFDEF OUTER",
+			"  LINE1",
+			"  #IFDEF INNER",
+			"    LINE2",
+			"  #IFEND",
+			"  LINE3",
+			"#IFEND",
+			"ALWAYS",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 3 {
+		t.Fatalf("expected 3 lines, got %d", len(lines))
+	}
+
+	expected := []string{"LINE1", "LINE3", "ALWAYS"}
+	for i, exp := range expected {
+		if !strings.Contains(lines[i].Text, exp) {
+			t.Errorf("line %d: expected %q, got %q", i, exp, lines[i].Text)
+		}
+	}
+}
+
+func TestPreProcess_ConditionalBlocking(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#IFNDEF UNDEFINED",
+			"  VISIBLE",
+			"  #DEFINE FOO = 1",
+			"#IFEND",
+			"#IFDEF UNDEFINED",
+			"  INVISIBLE",
+			"  #DEFINE BAR = 2",
+			"#IFEND",
+			"LDA #FOO",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 2 {
+		t.Fatalf("expected 2 lines, got %d", len(lines))
+	}
+
+	// FOO should be defined, BAR should not
+	if lines[1].Text != "LDA #1" {
+		t.Errorf("expected 'LDA #1', got %q", lines[1].Text)
+	}
+}
+
+func TestPreProcess_NestedIncludes(t *testing.T) {
+	files := map[string][]string{
+		"a.c65": {
+			"#INCLUDE b.c65",
+			"LINE_A",
+		},
+		"b.c65": {
+			"#INCLUDE c.c65",
+			"LINE_B",
+		},
+		"c.c65": {
+			"LINE_C",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("a.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 3 {
+		t.Fatalf("expected 3 lines, got %d", len(lines))
+	}
+
+	expected := []string{"LINE_C", "LINE_B", "LINE_A"}
+	for i, exp := range expected {
+		if lines[i].Text != exp {
+			t.Errorf("line %d: expected %q, got %q", i, exp, lines[i].Text)
+		}
+	}
+}
+
+func TestPreProcess_Filename(t *testing.T) {
+	files := map[string][]string{
+		"main.c65": {
+			"#INCLUDE lib.c65",
+			"MAIN_LINE",
+		},
+		"lib.c65": {
+			"LIB_LINE",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("main.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if lines[0].Filename != "lib.c65" {
+		t.Errorf("expected filename 'lib.c65', got %q", lines[0].Filename)
+	}
+	if lines[1].Filename != "main.c65" {
+		t.Errorf("expected filename 'main.c65', got %q", lines[1].Filename)
+	}
+}
+
+func TestPreProcess_LineNumbers(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"LINE1",
+			"LINE2",
+			"#DEFINE FOO = 1",
+			"LINE4",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	expectedLines := []int{1, 2, 4}
+	for i, exp := range expectedLines {
+		if lines[i].LineNo != exp {
+			t.Errorf("line %d: expected line number %d, got %d", i, exp, lines[i].LineNo)
+		}
+	}
+}
+
+func TestPreProcess_Undef(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#DEFINE FOO = 1",
+			"LDA #FOO",
+			"#UNDEF FOO",
+			"LDA #FOO",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if lines[0].Text != "LDA #1" {
+		t.Errorf("expected 'LDA #1', got %q", lines[0].Text)
+	}
+	if lines[1].Text != "LDA #FOO" {
+		t.Errorf("expected 'LDA #FOO' (unexpanded), got %q", lines[1].Text)
+	}
+}
+
+func TestPreProcess_PragmaWithDefines(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#DEFINE SETTING = 1",
+			"#PRAGMA USE_FEATURE SETTING",
+			"CODE",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	// Pragma should have been processed with define expansion
+	if len(lines) != 1 {
+		t.Fatalf("expected 1 line, got %d", len(lines))
+	}
+}
+
+func TestPreProcess_EmptyFile(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 0 {
+		t.Errorf("expected 0 lines, got %d", len(lines))
+	}
+}
+
+func TestPreProcess_MissingInclude(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#INCLUDE notfound.c65",
+		},
+	}
+	reader := NewMockFileReader(files)
+	_, err := PreProcess("test.c65", reader)
+	if err == nil {
+		t.Error("expected error for missing include")
+	}
+}
+
+func TestPreProcess_HaltDirective(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"LINE1",
+			"#HALT",
+			"LINE2",
+		},
+	}
+	reader := NewMockFileReader(files)
+	_, err := PreProcess("test.c65", reader)
+	if err == nil {
+		t.Error("expected HaltError")
+	}
+	if _, ok := err.(HaltError); !ok {
+		t.Errorf("expected HaltError, got %T", err)
+	}
+}
+
+func TestPreProcess_ConditionalHalt(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"LINE1",
+			"#IFDEF UNDEFINED",
+			"#HALT",
+			"#IFEND",
+			"LINE2",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+	// HALT is inside false conditional, should not trigger
+	if len(lines) != 2 {
+		t.Errorf("expected 2 lines, got %d", len(lines))
+	}
+}
+
+func TestPreProcess_Tokens(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"LDA #$42",
+			"  STA   $D020  ",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	expectedTokens := [][]string{
+		{"LDA", "#$42"},
+		{"STA", "$D020"},
+	}
+
+	for i, expected := range expectedTokens {
+		if len(lines[i].Tokens) != len(expected) {
+			t.Errorf("line %d: expected %d tokens, got %d", i, len(expected), len(lines[i].Tokens))
+			continue
+		}
+		for j, tok := range expected {
+			if lines[i].Tokens[j] != tok {
+				t.Errorf("line %d token %d: expected %q, got %q", i, j, tok, lines[i].Tokens[j])
+			}
+		}
+	}
+}
+
+func TestPreProcess_ASMTokens(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"ASM",
+			"  lda #$42",
+			"ENDASM",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	// ASM and ENDASM have empty token arrays
+	if len(lines[0].Tokens) != 0 {
+		t.Errorf("ASM should have empty tokens, got %d", len(lines[0].Tokens))
+	}
+	if len(lines[1].Tokens) != 0 {
+		t.Errorf("ASM content should have empty tokens, got %d", len(lines[1].Tokens))
+	}
+	if len(lines[2].Tokens) != 0 {
+		t.Errorf("ENDASM should have empty tokens, got %d", len(lines[2].Tokens))
+	}
+}
+
+func TestPreProcess_ComplexIncludeGuard(t *testing.T) {
+	files := map[string][]string{
+		"c64scr.c65": {
+			"#IFNDEF __C64_SCR",
+			"#DEFINE __C64_SCR = 1",
+			"GOTO lib_c64scr_skip",
+			"LABEL lib_c64scr_blank",
+			"ASM",
+			"  lda $d011",
+			"ENDASM",
+			"SUBEND",
+			"LABEL lib_c64scr_skip",
+			"#IFEND",
+		},
+		"main.c65": {
+			"#INCLUDE c64scr.c65",
+			"#INCLUDE c64scr.c65",
+			"MAIN",
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("main.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	// First include should emit the library, second should be blocked
+	// Count non-directive lines from first include
+	count := 0
+	for _, line := range lines {
+		if strings.Contains(line.Text, "lda $d011") {
+			count++
+		}
+	}
+
+	if count != 1 {
+		t.Errorf("expected 1 lib lines from multiple includes of same file, got %d", count)
+	}
+}
+
+func TestPreProcess_UnbalancedConditionals(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"#DEFINE FOO",
+			"#IFDEF FOO",
+			"  LINE1",
+			"  LINE2",
+			// Missing #IFEND
+		},
+	}
+	reader := NewMockFileReader(files)
+	_, err := PreProcess("test.c65", reader)
+	if err == nil {
+		t.Error("expected error for unbalanced conditionals")
+	}
+	if !strings.Contains(err.Error(), "unbalanced") {
+		t.Errorf("expected 'unbalanced' error, got: %v", err)
+	}
+}
+
+func TestPreProcess_FilenameAndLineNumberTracking(t *testing.T) {
+	files := map[string][]string{
+		"main.c65": {
+			"MAIN_LINE1",       // line 1
+			"#INCLUDE lib.c65", // line 2 (directive, not emitted)
+			"MAIN_LINE2",       // line 3
+			"MAIN_LINE3",       // line 4
+		},
+		"lib.c65": {
+			"LIB_LINE1", // line 1
+			"LIB_LINE2", // line 2
+			"LIB_LINE3", // line 3
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("main.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	expected := []struct {
+		text     string
+		filename string
+		lineNo   int
+	}{
+		{"MAIN_LINE1", "main.c65", 1},
+		{"LIB_LINE1", "lib.c65", 1},
+		{"LIB_LINE2", "lib.c65", 2},
+		{"LIB_LINE3", "lib.c65", 3},
+		{"MAIN_LINE2", "main.c65", 3},
+		{"MAIN_LINE3", "main.c65", 4},
+	}
+
+	if len(lines) != len(expected) {
+		t.Fatalf("expected %d lines, got %d", len(expected), len(lines))
+	}
+
+	for i, exp := range expected {
+		if lines[i].Text != exp.text {
+			t.Errorf("line %d: expected text %q, got %q", i, exp.text, lines[i].Text)
+		}
+		if lines[i].Filename != exp.filename {
+			t.Errorf("line %d: expected filename %q, got %q", i, exp.filename, lines[i].Filename)
+		}
+		if lines[i].LineNo != exp.lineNo {
+			t.Errorf("line %d: expected line number %d, got %d", i, exp.lineNo, lines[i].LineNo)
+		}
+	}
+}
+
+func TestPreProcess_Tokenization(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"LDA #$42",                    // simple tokens
+			"  STA   $D020  ",             // leading/trailing whitespace, multiple spaces
+			"LET var = $1000",             // multiple tokens with =
+			"	JMP	$0810",                  // tabs
+			"  CALL  func  (  a  b  c  )", // spaces around everything
+			"",                            // empty line
+			"NOP",                         // single token
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	expected := [][]string{
+		{"LDA", "#$42"},
+		{"STA", "$D020"},
+		{"LET", "var", "=", "$1000"},
+		{"JMP", "$0810"},
+		{"CALL", "func", "(", "a", "b", "c", ")"},
+		{}, // empty line has no tokens
+		{"NOP"},
+	}
+
+	if len(lines) != len(expected) {
+		t.Fatalf("expected %d lines, got %d", len(expected), len(lines))
+	}
+
+	for i, exp := range expected {
+		if len(lines[i].Tokens) != len(exp) {
+			t.Errorf("line %d: expected %d tokens, got %d (tokens: %v)",
+				i, len(exp), len(lines[i].Tokens), lines[i].Tokens)
+			continue
+		}
+		for j, tok := range exp {
+			if lines[i].Tokens[j] != tok {
+				t.Errorf("line %d token %d: expected %q, got %q", i, j, tok, lines[i].Tokens[j])
+			}
+		}
+	}
+}
+
+func TestPreProcess_PragmaTracking(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"LINE1",                     // no pragmas yet (index 0)
+			"LINE2",                     // no pragmas yet (index 0)
+			"#PRAGMA FEATURE1 enabled",  // directive, not emitted
+			"LINE3",                     // FEATURE1=enabled (index 1)
+			"LINE4",                     // FEATURE1=enabled (index 1)
+			"#PRAGMA FEATURE2 123",      // directive, not emitted
+			"LINE5",                     // FEATURE1=enabled, FEATURE2=123 (index 2)
+			"#PRAGMA FEATURE1 disabled", // directive, not emitted
+			"LINE6",                     // FEATURE1=disabled, FEATURE2=123 (index 3)
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+	if err != nil {
+		t.Fatalf("PreProcess failed: %v", err)
+	}
+
+	if len(lines) != 6 {
+		t.Fatalf("expected 6 lines, got %d", len(lines))
+	}
+
+	// Create pragma instance to verify the indices
+	pragma := NewPragma()
+	pragma.AddPragma("FEATURE1", "enabled")
+	pragma.AddPragma("FEATURE2", "123")
+	pragma.AddPragma("FEATURE1", "disabled")
+
+	tests := []struct {
+		lineIdx      int
+		text         string
+		pragmaSetIdx int
+		feature1     string
+		feature2     string
+	}{
+		{0, "LINE1", 0, "", ""},
+		{1, "LINE2", 0, "", ""},
+		{2, "LINE3", 1, "enabled", ""},
+		{3, "LINE4", 1, "enabled", ""},
+		{4, "LINE5", 2, "enabled", "123"},
+		{5, "LINE6", 3, "disabled", "123"},
+	}
+
+	for _, tt := range tests {
+		line := lines[tt.lineIdx]
+
+		if line.Text != tt.text {
+			t.Errorf("line %d: expected text %q, got %q", tt.lineIdx, tt.text, line.Text)
+		}
+
+		if line.PragmaSetIndex != tt.pragmaSetIdx {
+			t.Errorf("line %d: expected pragma set index %d, got %d",
+				tt.lineIdx, tt.pragmaSetIdx, line.PragmaSetIndex)
+		}
+
+		// Verify we can retrieve pragma values using the index
+		pragmaSet := pragma.GetPragmaSetByIndex(line.PragmaSetIndex)
+
+		if got := pragmaSet.GetPragma("FEATURE1"); got != tt.feature1 {
+			t.Errorf("line %d: expected FEATURE1=%q, got %q", tt.lineIdx, tt.feature1, got)
+		}
+
+		if got := pragmaSet.GetPragma("FEATURE2"); got != tt.feature2 {
+			t.Errorf("line %d: expected FEATURE2=%q, got %q", tt.lineIdx, tt.feature2, got)
+		}
+	}
+}
+
+func TestPreProcess_Halt(t *testing.T) {
+	files := map[string][]string{
+		"test.c65": {
+			"LINE1",
+			"LINE2",
+			"#HALT",
+			"LINE3", // should not be emitted
+			"LINE4", // should not be emitted
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("test.c65", reader)
+
+	if err == nil {
+		t.Fatal("expected HaltError, got nil")
+	}
+
+	if _, ok := err.(HaltError); !ok {
+		t.Fatalf("expected HaltError, got %T: %v", err, err)
+	}
+
+	// Lines before HALT should still be processed
+	if len(lines) != 2 {
+		t.Errorf("expected 2 lines before halt, got %d", len(lines))
+	}
+
+	if len(lines) >= 1 && lines[0].Text != "LINE1" {
+		t.Errorf("expected LINE1, got %q", lines[0].Text)
+	}
+
+	if len(lines) >= 2 && lines[1].Text != "LINE2" {
+		t.Errorf("expected LINE2, got %q", lines[1].Text)
+	}
+}
+
+func TestPreProcess_HaltInInclude(t *testing.T) {
+	files := map[string][]string{
+		"main.c65": {
+			"MAIN_LINE1",
+			"#INCLUDE lib.c65",
+			"MAIN_LINE2", // should not be reached
+		},
+		"lib.c65": {
+			"LIB_LINE1",
+			"#HALT",
+			"LIB_LINE2", // should not be emitted
+		},
+	}
+	reader := NewMockFileReader(files)
+	lines, err := PreProcess("main.c65", reader)
+
+	if err == nil {
+		t.Fatal("expected HaltError, got nil")
+	}
+
+	if _, ok := err.(HaltError); !ok {
+		t.Fatalf("expected HaltError, got %T: %v", err, err)
+	}
+
+	// Should have MAIN_LINE1 and LIB_LINE1
+	if len(lines) != 2 {
+		t.Fatalf("expected 2 lines before halt, got %d", len(lines))
+	}
+
+	if lines[0].Text != "MAIN_LINE1" {
+		t.Errorf("expected MAIN_LINE1, got %q", lines[0].Text)
+	}
+
+	if lines[1].Text != "LIB_LINE1" {
+		t.Errorf("expected LIB_LINE1, got %q", lines[1].Text)
+	}
+}