class Print1To10 implements Runnable {
    public int sum = 0;

    public void run() {
        for (int i = 1; i <= 10; i++) {
            System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
            sum += i;
        }
    }
}

class PrintEven implements Runnable {
    public void run() {
        for (int i = 2; i <= 20; i += 2) {
            System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
        }
    }
}

class PrintOdd implements Runnable {
    public void run() {
        for (int i = 1; i < 20; i += 2) {
            System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
        }    
    }
}

class IncrementCounter {
    protected int counter = 0;
    public int incrementCount = 100000;

    public int getCounterValue() {
        return counter;
    }
}

class IncrementCounterUnsynchronized extends IncrementCounter implements Runnable{
    public void run() {
        for (int i = 0; i < incrementCount; i++) {
            counter++;
        }
    }
}

class IncrementCounterSynchronized extends IncrementCounter implements Runnable {
    private final Object lockCounter = new Object();
    
    public void run() {
        for (int i = 0; i < incrementCount; i++) {
            synchronized (lockCounter) {
                counter++;
            }
        }
    }
}

public class java_15ThreadsRaces {
    public static void main(String[] args) {
        Print1To10 print1To10 = new Print1To10();

        Thread print1To10Task = new Thread(print1To10);
        Thread printEvenTask = new Thread(new PrintEven());
        Thread printOddTask = new Thread(new PrintOdd());
        
        Thread print5Multiple = new Thread(() -> {
            for (int i = 5; i <= 50; i += 5) {
                System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
            }
        });

        print1To10Task.start();
        printEvenTask.start();
        printOddTask.start();
        print5Multiple.start();

        try {
            print1To10Task.join();
            printEvenTask.join();
            printOddTask.join();
            print5Multiple.join();
        } catch (InterruptedException e) {
            System.out.println("Exception Occurred: " + e);
        } 

        boolean print1To10TaskCompleted = print1To10Task.isAlive();
        boolean printEvenTaskCompleted = printEvenTask.isAlive();
        boolean printOddTaskCompleted = printOddTask.isAlive();
        boolean print5MultipleCompleted = print5Multiple.isAlive();

        System.out.println(
            !print1To10TaskCompleted && 
            !printEvenTaskCompleted &&
            !printOddTaskCompleted &&
            !print5MultipleCompleted ? "All tasks complete. Result of Print1To10Numbers: " + print1To10.sum : "Tasks still pending"
        );

        System.out.println();
        System.out.println("unsynchronized [5+ runs, showing some incorrect] Unsynchronized Run + ==================== +");
        for (int i = 0; i < 5; i++) {
            IncrementCounterUnsynchronized unsynchronizedIncrement = new IncrementCounterUnsynchronized();
            int incrementCountUnsynchronized = unsynchronizedIncrement.incrementCount;

            Thread unsyncThread1 = new Thread(unsynchronizedIncrement);
            Thread unsyncThread2 = new Thread(unsynchronizedIncrement);
            Thread unsyncThread3 = new Thread(unsynchronizedIncrement);
            Thread unsyncThread4 = new Thread(unsynchronizedIncrement);

            unsyncThread1.start();
            unsyncThread2.start();
            unsyncThread3.start();
            unsyncThread4.start();

            try {
                unsyncThread1.join();
                unsyncThread2.join();
                unsyncThread3.join();
                unsyncThread4.join();
            } catch (InterruptedException e) {
                System.out.println("Exception Occurred: " + e);
            }

            System.out.println("Counter Value Expected (4 threads times " + incrementCountUnsynchronized + "): " + 4 * incrementCountUnsynchronized);
            System.out.println("Counter Value Actual (Run " + (i+1) + "): " + unsynchronizedIncrement.getCounterValue());
        }
        System.out.println(" + ==================== + Unsynchronized Run ");

        // This works because synchronized locks another thread to interrupt 3 step process of increment
        // ReentrantLock was not used due to not requiring much of the methods available by it and synchronized can handle this task efficiently
        System.out.println("\nfixed [5+ runs, all exactly correct] Synchronized Run + ==================== +");
        for (int i = 0; i < 5; i++) {
            IncrementCounterSynchronized synchronizedIncrement = new IncrementCounterSynchronized();
            int incrementCountSynchronized = synchronizedIncrement.incrementCount;

            Thread syncThread1 = new Thread(synchronizedIncrement);
            Thread syncThread2 = new Thread(synchronizedIncrement);
            Thread syncThread3 = new Thread(synchronizedIncrement);
            Thread syncThread4 = new Thread(synchronizedIncrement);

            syncThread1.start();
            syncThread2.start();
            syncThread3.start();
            syncThread4.start();

            try {
                syncThread1.join();
                syncThread2.join();
                syncThread3.join();
                syncThread4.join();
            } catch (InterruptedException e) {
                System.out.println("Exception Occurred: " + e);
            }

            System.out.println("Counter Value Expected (4 threads times " + incrementCountSynchronized + "): " + 4 * incrementCountSynchronized);
            System.out.println("Counter Value Actual (Run " + (i+1) + "): " + synchronizedIncrement.getCounterValue());
        }
        System.out.println(" + ==================== + Synchronized Run ");

        final Object lockA = new Object();
        final Object lockB = new Object();

        System.out.println();
        System.out.println("=== 1. DEMONSTRATING DEADLOCK ===");
        Thread deadlockIncorrectA = new Thread(() -> {
            try {
                synchronized (lockA) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock A");
                    Thread.sleep(50);
                    
                    synchronized (lockB) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock B");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        Thread deadlockIncorrectB = new Thread(() -> {
            try {
                synchronized (lockB) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock B");
                    Thread.sleep(50);
                    
                    synchronized (lockA) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock A");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        deadlockIncorrectA.setDaemon(true);
        deadlockIncorrectB.setDaemon(true);

        deadlockIncorrectA.start();
        deadlockIncorrectB.start();

        try {
            deadlockIncorrectA.join(2000);
            deadlockIncorrectB.join(2000);
        } catch (InterruptedException e) {
            System.out.println("Exception Occured: " + e);
        } 

        boolean thread1Alive = deadlockIncorrectA.isAlive();
        boolean thread2Alive = deadlockIncorrectB.isAlive();
        
        System.out.println("Timeout expired. Thread 1 alive? " + thread1Alive);
        System.out.println("Timeout expired. Thread 2 alive? " + thread2Alive);
        System.out.println(thread1Alive && thread2Alive ? "-> Result: Deadlock successfully occurred!\n" : "-> Result: Threads finished.\n");

        final Object lockC = new Object();
        final Object lockD = new Object();
        // NOTE: not using locks A and B as they are deadlocked until main is completed,
        // Therefore using the same locks will show the same output and not provide a solution to deadlock

        System.out.println("=== 2. FIXING THE DEADLOCK (Consistent Order) ===");
        Thread deadlockCorrectA = new Thread(() -> {
            try {
                synchronized (lockC) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock C");
                    Thread.sleep(50);
                    
                    synchronized (lockD) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock D");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        Thread deadlockCorrectB = new Thread(() -> {
            try {
                synchronized (lockC) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock C");
                    Thread.sleep(50);
                    
                    synchronized (lockD) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock D");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        long startTime = System.currentTimeMillis();

        deadlockCorrectA.start();
        deadlockCorrectB.start();

        try {
            deadlockCorrectA.join(2000);
            deadlockCorrectB.join(2000);
        } catch (InterruptedException e) {
            System.out.println("Exception Occured: " + e);
        } 

        long elapsedTime = System.currentTimeMillis() - startTime;
        boolean fixedThread1Alive = deadlockCorrectA.isAlive();
        boolean fixedThread2Alive = deadlockCorrectB.isAlive();
        
        System.out.println("Elapsed time: " + elapsedTime + " ms");
        System.out.println("Timeout expired. Thread 1 alive? " + fixedThread1Alive);
        System.out.println("Timeout expired. Thread 2 alive? " + fixedThread2Alive);
        System.out.println(fixedThread1Alive && fixedThread2Alive ? "-> Result: Deadlock successfully occurred!\n" : "-> Result: Threads finished.\n");
    }
}

Today’s program is short, but nearly every line in it answers a question that computer scientists were asking before most programmers alive today were born. Why can two correct pieces of code produce a wrong answer together? What does it mean to take turns safely? What exactly has to be true for two threads to freeze each other?

This article is the theory companion to the practical walkthrough of today’s code. Last lesson, we looked at the theory behind lazy evaluation, folds, and Optional, and the Streams guide showed it in code. Streams were about values flowing through a pipeline. Today is about what happens when several flows touch the same value at the same time.

Sequential Processes and the Problem of Interleaving

Why the output of four threads has no fixed order

In the 1960s, Edsger W. Dijkstra’s essay “Cooperating Sequential Processes” (EWD123, 1965) framed concurrent programs as a set of ordinary sequential processes that occasionally need to coordinate. A key idea was that the relative speeds of those processes are unknown and arbitrary. A correct program may not depend on one process being faster than another.

That idea is what the first block of our program demonstrates:

print1To10Task.start();
printEvenTask.start();
printOddTask.start();
print5Multiple.start();

Each Thread is a sequential process. Within Thread-1, 2 is always printed before 4. Across threads, the JVM scheduler and the operating system decide who runs when, so the lines may appear grouped or mixed, and either is a valid outcome. Programs that only work because one thread happened to be quicker are exactly what Dijkstra's principle rules out.

The same line of thinking explains why join() exists. It is the point where one process waits for another to finish, so main can safely read print1To10.sum afterwards.

Critical Sections and Mutual Exclusion

The problem that gave counter++ its danger

In 1965, Dijkstra published “Solution of a Problem in Concurrent Programming Control” in Communications of the ACM (volume 8, issue 9). The problem it stated is one that still defines the field: several processes, each with a stretch of code that must never run in two places at once. That stretch is called a critical section, and keeping processes out of each other’s critical sections is called mutual exclusion. Dijkstra’s paper gave a solution using only ordinary reads and writes of shared variables.

In our code, the critical section is one line:

for (int i = 0; i < incrementCount; i++) {
    counter++;
}

The line looks tiny, but it contains three separate actions: read counter, add one, write the result back. If two threads are between "read" and "write" at the same moment, each overwrites the other's work. In the language of the theory, the critical section was not protected, so the final total is no longer the sum of the work done.

Races, Atomicity, and the Java Memory Model

Why 400,000 became 135,072

One recorded run of the unsynchronized version produced this:

Counter Value Expected (4 threads times 100000): 400000
Counter Value Actual (Run 1): 135072

Brian Goetz and co-authors, in Java Concurrency in Practice (Addison-Wesley, 2006), call this pattern a read-modify-write sequence and explain that it is a race condition: the result depends on the unlucky timing of threads. The fix needs the three steps to be atomic, meaning indivisible from the viewpoint of every other thread.

There is a second, deeper problem hiding in this code. Chapter 17 of the Java Language Specification defines what a thread is guaranteed to see when another thread writes a value. It does this through a relation called happens-before, a term borrowed from Leslie Lamport’s 1978 paper “Time, Clocks, and the Ordering of Events in a Distributed System” in Communications of the ACM. The Java Memory Model itself is described by Jeremy Manson, William Pugh, and Sarita Adve in “The Java Memory Model” (POPL 2005). When two threads access the same variable, at least one access is a write, and nothing orders the accesses by happens-before, the specification calls it a data race. Our unprotected counter is exactly that. It is not only that the arithmetic can be lost, but that one thread is not even promised to see another's latest write.

Some of that ordering is provided for free. The specification states that all actions in a thread happen-before another thread successfully returns from join() on it. That is why this works:

print1To10Task.join();
// ...
System.out.println("... " + print1To10.sum);

The thread wrote sum, and join() creates the ordering that makes that write visible to main.

Monitors: Turning a Rule into a Language Feature

What synchronized really is

C. A. R. Hoare’s “Monitors: An Operating System Structuring Concept” (Communications of the ACM, October 1974, pp. 549–557) proposed gathering shared data together with the only code allowed to touch it, and having the system guarantee that one process at a time is inside. Per Brinch Hansen was developing the same idea at the same time, and the two are usually credited together. The point was to stop relying on programmers remembering to guard every access by hand.

Java adopted the monitor idea. The Java Language Specification, in its chapter on threads and locks, describes synchronization as implemented using monitors, with one monitor associated with every object. Our fix uses exactly that:

private final Object lockCounter = new Object();

public void run() {
    for (int i = 0; i < incrementCount; i++) {
        synchronized (lockCounter) {
            counter++;
        }
    }
}

Entering the block acquires the monitor of lockCounter, and leaving it releases the monitor, even if an exception is thrown. This is where Java differs from Hoare's original design: Hoare bound a monitor to a data structure, whereas Java lets any object serve as the lock, and the programmer chooses which one. That freedom is also a responsibility. Protection only exists if every thread contending for the data uses the same lock object, which is why all four threads in our run share a single IncrementCounterSynchronized instance.

The monitor also carries the memory guarantee from the previous section. In the specification’s happens-before rules, releasing a monitor happens-before every later acquisition of the same monitor. So the lock does two jobs at once: it provides mutual exclusion, and it makes each thread’s write visible to the next. That second job is why every one of our synchronized runs reports exactly 400,000.

Deadlock and the Four Coffman Conditions

Exactly what has to be true for two threads to freeze

In 1971, E. G. Coffman, M. J. Elphick, and A. Shoshani published “System Deadlocks” in ACM Computing Surveys (volume 3, issue 2, pages 67–78). They identified four conditions that must all hold at the same time for a deadlock to occur. Each of them can be pointed to in the deadlock demo:

• Mutual exclusion: a lock can be held by only one thread. This is the synchronized block itself.
• Hold and wait: a thread holds one resource while waiting for another. Thread A holds lockA while it asks for lockB, and the Thread.sleep(50) makes sure that both threads reach this state.
• No preemption: a resource cannot be taken away from its holder. A Java monitor is released only when the holding thread leaves its block, and a thread stuck waiting to enter a synchronized block cannot be talked out of it by an interrupt.
• Circular wait: there is a closed chain of threads, each waiting for a resource held by the next. A waits for B’s lock, and B waits for A’s.

// Thread A                          // Thread B
synchronized (lockA) {               synchronized (lockB) {
    Thread.sleep(50);                    Thread.sleep(50);
    synchronized (lockB) { ... }         synchronized (lockA) { ... }
}                                    }

The theory also explains the shape of the classic puzzle. Dijkstra set the dining philosophers problem as an exam question in 1965, and Hoare later gave it the form we know today: five philosophers, five forks, and each philosopher needs two forks to eat. If every philosopher picks up the left fork first, all of them can end up holding one fork and waiting forever. It is our two-lock program with more threads. Dijkstra’s treatment appeared in “Hierarchical Ordering of Sequential Processes” (Acta Informatica, volume 1, issue 2, 1971, pp. 115–138).

Breaking the Cycle: Why Lock Ordering Works

Removing one condition is enough

Because a deadlock needs all four conditions, removing any one of them prevents it. We cannot give up mutual exclusion, because that would bring back the race. We cannot cheaply give up hold-and-wait or no-preemption with plain synchronized. The practical condition to attack is circular wait, and the standard way is resource ordering: give every lock a rank and require everyone to acquire locks in rank order.

// Both threads now take lockC first, then lockD
synchronized (lockC) {
    Thread.sleep(50);
    synchronized (lockD) { ... }
}

If every thread follows the same order, a cycle cannot form. A thread holding lockD can never be waiting for lockC, because no thread ever asks for lockC while already holding lockD. This ordering discipline is also the resource-hierarchy solution to the dining philosophers problem, and Java Concurrency in Practice devotes a chapter on liveness hazards to lock-ordering deadlocks.

There is a price. The two threads now run one after the other through the locked section, and the measured 102 ms is the two sleep(50) calls back to back. Prevention trades some concurrency for the guarantee that the system cannot freeze. For our program that is clearly worth it.

Proving a Deadlock Without Living Through It

Timeouts, daemon threads, and the limits of detection

The program has to prove a deadlock happened without hanging forever, and it does so in two ways:

deadlockIncorrectA.setDaemon(true);
deadlockIncorrectB.setDaemon(true);
// ...
deadlockIncorrectA.join(2000);
deadlockIncorrectB.join(2000);
System.out.println("Timeout expired. Thread 1 alive? " + deadlockIncorrectA.isAlive());

The Java Language Specification’s section on program exit says the JVM exits once every non-daemon thread has finished. By marking the stuck threads as daemons, main can finish and let the program end. The timed join and isAlive() are a practical detection method: if both threads are still alive after two seconds, they are blocked.

This is a timeout, not a proof. A slow thread and a deadlocked thread look identical from outside, which is why the theory prefers to detect the cycle itself. For monitor locks, the JVM can do this: a thread dump from jstack, or the ThreadMXBean API, can report that threads are deadlocked and name the locks involved.

Putting It Together

Each idea, its origin, and the line of code that carries it

Interleaving with unknown speeds comes from Dijkstra’s sequential-process model, and it shows up in the four start() calls. Critical sections and mutual exclusion come from Dijkstra's 1965 work, and they show up in counter++. Atomicity, data races, and happens-before come from the Java Language Specification and the Java Memory Model research, and they show up in the result 135072 and in the join() before sum is read. Monitors come from Hoare and Brinch Hansen, and they show up in synchronized (lockCounter). Deadlock and its four conditions come from Coffman, Elphick, and Shoshani, and they show up in lockA and lockB. Resource ordering comes from the dining philosophers literature, and it shows up in lockC and lockD.

The recurring theme of this series, behavior that is silently wrong, is at its sharpest here. A race condition raises no exception. A passing run proves nothing. The theory gives us a way to reason about correctness that does not depend on getting lucky.

Further Reading

Dijkstra, E. W. “Cooperating Sequential Processes” (EWD123), 1965.

Dijkstra, E. W. “Solution of a Problem in Concurrent Programming Control.” Communications of the ACM 8(9), 1965.

Dijkstra, E. W. “Hierarchical Ordering of Sequential Processes.” Acta Informatica 1(2), 1971, pp. 115–138.

Coffman, E. G., Elphick, M. J., and Shoshani, A. “System Deadlocks.” ACM Computing Surveys 3(2), 1971, pp. 67–78.

Hoare, C. A. R. “Monitors: An Operating System Structuring Concept.” Communications of the ACM, October 1974, pp. 549–557.

Lamport, L. “Time, Clocks, and the Ordering of Events in a Distributed System.” Communications of the ACM, 1978.

Manson, J., Pugh, W., and Adve, S. “The Java Memory Model.” POPL 2005.

Goetz, B. et al. Java Concurrency in Practice. Addison-Wesley, 2006.

The Java Language Specification, chapters 12 (program exit) and 17 (threads and locks).

The Arbiter Journey So Far

This is lesson 15 of the Arbiter Learning Journey, a Java-from-scratch series building toward Arbiter, a microservices-based test management platform. Every concept is written and tested by hand before the next one begins. A test platform runs many things at once, so the ideas in this article, safe shared state and lock discipline, will matter every time Arbiter’s services handle more than one request.

Understand the concept in detail

Arbiter YouTube Playlist

Arbiter on GitHub

GitHub - Someone-anon-coder/Arbiter: Welcome to Arbiter - a documented, hands-on journey from Java fundamentals to a deployed microservices platform.

Arbiter Lessons

• List: Arbiter (Java) | Curated by Someone | Medium
• List: Arbiter (Java) Theory | Curated by Someone | Medium

Other series on this account

• List: Cybersecurity | Curated by Someone | Medium
• List: Python Guide | Curated by Someone | Medium
• List: Python Guide - Theory | Curated by Someone | Medium
• List: GoLang Guide | Curated by Someone | Medium
• List: GoLang Guide Theoretical | Curated by Someone | Medium
• List: My Progress - Matplotlib | Curated by Someone | Medium

What’s Next

Our program created 48 Thread objects by hand. The next lesson hands that job to the JVM with ExecutorService and thread pools, followed by CompletableFuture for results that arrive later.

Understanding Java Concurrency: The Theory Behind Threads, Race Conditions, Monitors, and Deadlock was originally published in CodeX on Medium, where people are continuing the conversation by highlighting and responding to this story.

class Print1To10 implements Runnable {
    public int sum = 0;

    public void run() {
        for (int i = 1; i <= 10; i++) {
            System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
            sum += i;
        }
    }
}

class PrintEven implements Runnable {
    public void run() {
        for (int i = 2; i <= 20; i += 2) {
            System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
        }
    }
}

class PrintOdd implements Runnable {
    public void run() {
        for (int i = 1; i < 20; i += 2) {
            System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
        }    
    }
}

class IncrementCounter {
    protected int counter = 0;
    public int incrementCount = 100000;

    public int getCounterValue() {
        return counter;
    }
}

class IncrementCounterUnsynchronized extends IncrementCounter implements Runnable{
    public void run() {
        for (int i = 0; i < incrementCount; i++) {
            counter++;
        }
    }
}

class IncrementCounterSynchronized extends IncrementCounter implements Runnable {
    private final Object lockCounter = new Object();
    
    public void run() {
        for (int i = 0; i < incrementCount; i++) {
            synchronized (lockCounter) {
                counter++;
            }
        }
    }
}

public class java_15ThreadsRaces {
    public static void main(String[] args) {
        Print1To10 print1To10 = new Print1To10();

        Thread print1To10Task = new Thread(print1To10);
        Thread printEvenTask = new Thread(new PrintEven());
        Thread printOddTask = new Thread(new PrintOdd());
        
        Thread print5Multiple = new Thread(() -> {
            for (int i = 5; i <= 50; i += 5) {
                System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
            }
        });

        print1To10Task.start();
        printEvenTask.start();
        printOddTask.start();
        print5Multiple.start();

        try {
            print1To10Task.join();
            printEvenTask.join();
            printOddTask.join();
            print5Multiple.join();
        } catch (InterruptedException e) {
            System.out.println("Exception Occurred: " + e);
        } 

        boolean print1To10TaskCompleted = print1To10Task.isAlive();
        boolean printEvenTaskCompleted = printEvenTask.isAlive();
        boolean printOddTaskCompleted = printOddTask.isAlive();
        boolean print5MultipleCompleted = print5Multiple.isAlive();

        System.out.println(
            !print1To10TaskCompleted && 
            !printEvenTaskCompleted &&
            !printOddTaskCompleted &&
            !print5MultipleCompleted ? "All tasks complete. Result of Print1To10Numbers: " + print1To10.sum : "Tasks still pending"
        );

        System.out.println();
        System.out.println("unsynchronized [5+ runs, showing some incorrect] Unsynchronized Run + ==================== +");
        for (int i = 0; i < 5; i++) {
            IncrementCounterUnsynchronized unsynchronizedIncrement = new IncrementCounterUnsynchronized();
            int incrementCountUnsynchronized = unsynchronizedIncrement.incrementCount;

            Thread unsyncThread1 = new Thread(unsynchronizedIncrement);
            Thread unsyncThread2 = new Thread(unsynchronizedIncrement);
            Thread unsyncThread3 = new Thread(unsynchronizedIncrement);
            Thread unsyncThread4 = new Thread(unsynchronizedIncrement);

            unsyncThread1.start();
            unsyncThread2.start();
            unsyncThread3.start();
            unsyncThread4.start();

            try {
                unsyncThread1.join();
                unsyncThread2.join();
                unsyncThread3.join();
                unsyncThread4.join();
            } catch (InterruptedException e) {
                System.out.println("Exception Occurred: " + e);
            }

            System.out.println("Counter Value Expected (4 threads times " + incrementCountUnsynchronized + "): " + 4 * incrementCountUnsynchronized);
            System.out.println("Counter Value Actual (Run " + (i+1) + "): " + unsynchronizedIncrement.getCounterValue());
        }
        System.out.println(" + ==================== + Unsynchronized Run ");

        // This works because synchronized locks another thread to interrupt 3 step process of increment
        // ReentrantLock was not used due to not requiring much of the methods available by it and synchronized can handle this task efficiently
        System.out.println("\nfixed [5+ runs, all exactly correct] Synchronized Run + ==================== +");
        for (int i = 0; i < 5; i++) {
            IncrementCounterSynchronized synchronizedIncrement = new IncrementCounterSynchronized();
            int incrementCountSynchronized = synchronizedIncrement.incrementCount;

            Thread syncThread1 = new Thread(synchronizedIncrement);
            Thread syncThread2 = new Thread(synchronizedIncrement);
            Thread syncThread3 = new Thread(synchronizedIncrement);
            Thread syncThread4 = new Thread(synchronizedIncrement);

            syncThread1.start();
            syncThread2.start();
            syncThread3.start();
            syncThread4.start();

            try {
                syncThread1.join();
                syncThread2.join();
                syncThread3.join();
                syncThread4.join();
            } catch (InterruptedException e) {
                System.out.println("Exception Occurred: " + e);
            }

            System.out.println("Counter Value Expected (4 threads times " + incrementCountSynchronized + "): " + 4 * incrementCountSynchronized);
            System.out.println("Counter Value Actual (Run " + (i+1) + "): " + synchronizedIncrement.getCounterValue());
        }
        System.out.println(" + ==================== + Synchronized Run ");

        final Object lockA = new Object();
        final Object lockB = new Object();

        System.out.println();
        System.out.println("=== 1. DEMONSTRATING DEADLOCK ===");
        Thread deadlockIncorrectA = new Thread(() -> {
            try {
                synchronized (lockA) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock A");
                    Thread.sleep(50);
                    
                    synchronized (lockB) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock B");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        Thread deadlockIncorrectB = new Thread(() -> {
            try {
                synchronized (lockB) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock B");
                    Thread.sleep(50);
                    
                    synchronized (lockA) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock A");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        deadlockIncorrectA.setDaemon(true);
        deadlockIncorrectB.setDaemon(true);

        deadlockIncorrectA.start();
        deadlockIncorrectB.start();

        try {
            deadlockIncorrectA.join(2000);
            deadlockIncorrectB.join(2000);
        } catch (InterruptedException e) {
            System.out.println("Exception Occured: " + e);
        } 

        boolean thread1Alive = deadlockIncorrectA.isAlive();
        boolean thread2Alive = deadlockIncorrectB.isAlive();
        
        System.out.println("Timeout expired. Thread 1 alive? " + thread1Alive);
        System.out.println("Timeout expired. Thread 2 alive? " + thread2Alive);
        System.out.println(thread1Alive && thread2Alive ? "-> Result: Deadlock successfully occurred!\n" : "-> Result: Threads finished.\n");

        final Object lockC = new Object();
        final Object lockD = new Object();
        // NOTE: not using locks A and B as they are deadlocked until main is completed,
        // Therefore using the same locks will show the same output and not provide a solution to deadlock

        System.out.println("=== 2. FIXING THE DEADLOCK (Consistent Order) ===");
        Thread deadlockCorrectA = new Thread(() -> {
            try {
                synchronized (lockC) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock C");
                    Thread.sleep(50);
                    
                    synchronized (lockD) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock D");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        Thread deadlockCorrectB = new Thread(() -> {
            try {
                synchronized (lockC) {
                    System.out.println(Thread.currentThread().getName() + " acquired Lock C");
                    Thread.sleep(50);
                    
                    synchronized (lockD) {
                        System.out.println(Thread.currentThread().getName() + " acquired Lock D");
                    }
                }
            } catch (InterruptedException e) {
                System.out.println("Exception Occured: " + e);
            }
        });

        long startTime = System.currentTimeMillis();

        deadlockCorrectA.start();
        deadlockCorrectB.start();

        try {
            deadlockCorrectA.join(2000);
            deadlockCorrectB.join(2000);
        } catch (InterruptedException e) {
            System.out.println("Exception Occured: " + e);
        } 

        long elapsedTime = System.currentTimeMillis() - startTime;
        boolean fixedThread1Alive = deadlockCorrectA.isAlive();
        boolean fixedThread2Alive = deadlockCorrectB.isAlive();
        
        System.out.println("Elapsed time: " + elapsedTime + " ms");
        System.out.println("Timeout expired. Thread 1 alive? " + fixedThread1Alive);
        System.out.println("Timeout expired. Thread 2 alive? " + fixedThread2Alive);
        System.out.println(fixedThread1Alive && fixedThread2Alive ? "-> Result: Deadlock successfully occurred!\n" : "-> Result: Threads finished.\n");
    }
}

Until today, every program in this series has done one thing at a time. A line ran, finished, and the next line ran. That is a comfortable way to write code, and it is about to stop being true.

In the last lesson, we built lazy Streams pipelines with reduce and groupingBy. The theory behind them is in the companion article on lazy evaluation, folds, and Optional. Today we make Java do several things at once, and we meet the first bug in this series that never throws an exception, never prints a warning, and sometimes doesn't even happen.

The program above does four things:

• Runs four independent tasks on four threads at the same time
• Lets four threads increment one shared counter with no protection, and watches the total go wrong
• Fixes the counter with synchronized
• Builds a deadlock on purpose, proves it happened, then removes it with one rule

Thread and Runnable: Describing Work vs. Running It

In Java, a Runnable is a piece of work and a Thread is the worker that performs it. Describing the work is the easy part:

class PrintEven implements Runnable {
    public void run() {
        for (int i = 2; i <= 20; i += 2) {
            System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
        }
    }
}

Runnable has exactly one method, run(). Because it has only one abstract method, it can also be written as a lambda, the same arrow syntax we used with Streams:

Thread print5Multiple = new Thread(() -> {
    for (int i = 5; i <= 50; i += 5) {
        System.out.println("Thread Name: " + Thread.currentThread().getName() + ", task value: " + i);
    }
});

Our program creates four threads this way: Print1To10 (a class that also stores a sum), PrintEven, PrintOdd, and the lambda above. Thread.currentThread().getName() is how each line reveals which thread printed it.

The most important rule is the next one. Calling .start() creates a new thread of execution. Calling .run() directly just runs the method on the thread you are already on, like any ordinary method call. Nothing concurrent happens.

print1To10Task.start();
printEvenTask.start();
printOddTask.start();
print5Multiple.start();

The Output Order Is Not Yours to Control

Run this program a few times and the first block of output changes. In some runs Thread-0 prints all ten of its numbers before any other thread says a word. In other runs the lines from Thread-1, Thread-2, and Thread-3 are mixed together. The JVM and the operating system decide who runs when, and each task here is so short that one thread can finish before the next has been scheduled.

Both orderings are correct. The only guarantee we have is within a single thread: Thread-1 will always print 2 before 4 before 6. Across threads, there is no promised order.

Start Everything First, Join Afterwards

print1To10Task.start();
printEvenTask.start();
printOddTask.start();
print5Multiple.start();

print1To10Task.join();
printEvenTask.join();
printOddTask.join();
print5Multiple.join();

join() makes the calling thread, here main, wait until that thread's run() has returned. The order matters. All four start() calls come first, so all four threads are running before main waits for anything. If we wrote start(); join(); start(); join(); for each thread, every thread would have to finish before the next one began, and we would have rebuilt a sequential program with extra steps.

After the joins, main checks isAlive() on each thread and prints the sum that Print1To10 accumulated:

All tasks complete. Result of Print1To10Numbers: 55

That 55 is reliable because join() guarantees that everything the thread wrote is visible to main once join() returns.

The Counter That Lies

Now the interesting part. Four threads, each running the same loop 100,000 times, all incrementing one shared field:

class IncrementCounterUnsynchronized extends IncrementCounter implements Runnable {
    public void run() {
        for (int i = 0; i < incrementCount; i++) {
            counter++;
        }
    }
}

All four threads are handed the same IncrementCounterUnsynchronized object, so they share one counter. The expected result is 4 x 100,000 = 400,000. Here is what one recorded run produced:

Counter Value Expected (4 threads times 100000): 400000
Counter Value Actual (Run 1): 135072

Nothing crashed and no exception was thrown. The program simply reported a wrong number, confidently.

The reason is that counter++ looks like one operation but is three: read the current value, add one, write the result back. Two threads can read the same value before either writes. Both compute the same new value and both write it. Two increments happened, but the counter only moved by one. This is called a lost update.

There is a second, quieter lesson here. In the loop of five runs, some iterations come out exactly right by luck. A run that passes proves nothing. A race condition is a probability, not a certainty, which is also why it only shows up reliably at 400,000 increments and would be easy to miss at 10.

synchronized: One Thread at a Time

The fix gives every thread the same lock object and makes the increment happen inside it:

class IncrementCounterSynchronized extends IncrementCounter implements Runnable {
    private final Object lockCounter = new Object();

    public void run() {
        for (int i = 0; i < incrementCount; i++) {
            synchronized (lockCounter) {
                counter++;
            }
        }
    }
}

Every Java object has an invisible lock, called a monitor. Entering a synchronized (lockCounter) block means taking that monitor, and only one thread can hold it at a time. While one thread is inside, the others wait at the door. That makes read, add, and write behave as one indivisible step.

Two details are easy to miss. First, the lock only works because all four threads share the same instance, and so the same lockCounter. If each thread had its own object, each would have its own lock and nothing would be protected. Second, the block releases the lock when it exits, even if an exception is thrown inside it, so the lock cannot be forgotten.

We could have used ReentrantLock, but we only need basic mutual exclusion, and synchronized handles that with less code. The result across all five runs:

Counter Value Expected (4 threads times 100000): 400000
Counter Value Actual (Run 1): 400000
...
Counter Value Actual (Run 5): 400000

Deadlock: Two Locks, Opposite Order

A lock fixes the race, but locks create a new way to fail. Here two threads each need two locks, and they ask for them in opposite orders:

// Thread A                          // Thread B
synchronized (lockA) {               synchronized (lockB) {
    Thread.sleep(50);                    Thread.sleep(50);
    synchronized (lockB) { ... }         synchronized (lockA) { ... }
}                                    }

Thread A takes lockA, Thread B takes lockB, and the sleep(50) makes sure both have done so before either asks for its second lock. Now A waits for lockB, which B holds, and B waits for lockA, which A holds. Neither can leave its block to release what it holds until it has the second lock. They will wait forever.

A program that hangs forever is a bad way to prove a point, so the code uses two safety nets:

deadlockIncorrectA.setDaemon(true);
deadlockIncorrectB.setDaemon(true);

deadlockIncorrectA.start();
deadlockIncorrectB.start();

deadlockIncorrectA.join(2000);
deadlockIncorrectB.join(2000);

setDaemon(true) marks the threads as background threads, so the JVM can still exit when main finishes even though they are stuck. join(2000) waits at most two seconds instead of forever. After that, isAlive() is the evidence:

=== 1. DEMONSTRATING DEADLOCK ===
Thread-44 acquired Lock A
Thread-45 acquired Lock B
Timeout expired. Thread 1 alive? true
Timeout expired. Thread 2 alive? true
-> Result: Deadlock successfully occurred!

Notice that each thread printed that it got its first lock, and neither ever printed the second. Both still alive after the timeout means they are blocked, not just slow. (The thread numbers 44 and 45 are only a global counter, and they show how many threads this program had already created by that point.)

The Fix: One Order for Everyone

The cure is not fewer locks. It is the same order every time:

// Thread A                          // Thread B
synchronized (lockC) {               synchronized (lockC) {
    Thread.sleep(50);                    Thread.sleep(50);
    synchronized (lockD) { ... }         synchronized (lockD) { ... }
}                                    }

Both threads now take lockC first, then lockD. Whoever wins lockC makes the other wait at the first lock, and nobody can ever hold lockD while waiting for lockC, so a circular wait cannot form. We use fresh locks C and D on purpose: A and B are still held by the stuck daemon threads, so reusing them would just reproduce the deadlock.

=== 2. FIXING THE DEADLOCK (Consistent Order) ===
Thread-46 acquired Lock C
Thread-46 acquired Lock D
Thread-47 acquired Lock C
Thread-47 acquired Lock D
Elapsed time: 102 ms
Timeout expired. Thread 1 alive? false
Timeout expired. Thread 2 alive? false
-> Result: Threads finished.

The two threads now run one after the other instead of side by side, and the elapsed time of about 102 ms shows it: two sleep(50) calls back to back. Slightly sequential and correct is a far better outcome than concurrent and frozen. The thread names also show they never overlapped, with Thread-46 completing both locks before Thread-47 began.

Key Takeaways

• .start() creates a thread; .run() is just a method call
• Start all threads before joining any of them, or the work becomes sequential again
• Output order across threads is not guaranteed and changes between runs
• counter++ is three steps, and unprotected shared state can lose updates with no error at all
• A passing run does not prove a race condition is absent
• synchronized on one shared lock object makes the read-add-write step indivisible
• Two threads taking two locks in opposite orders can deadlock; taking them in the same order cannot
• join(timeout) plus isAlive() lets you prove a deadlock without hanging the program

The Arbiter Journey So Far

This is lesson 15 of the Arbiter Learning Journey, a Java-from-scratch series building toward Arbiter, a microservices-based test management platform. Every lesson comes with a runnable source file, and every concept is written and tested by hand before moving on. A test platform runs many things at once, such as test executions, notifications, and API requests, so today’s lesson is groundwork for what comes later.

Understand the concepts in detail

Watch the series on YouTube

Code on GitHub

GitHub - Someone-anon-coder/Arbiter: Welcome to Arbiter - a documented, hands-on journey from Java fundamentals to a deployed microservices platform.

Arbiter Lessons

• List: Arbiter (Java) | Curated by Someone | Medium
• List: Arbiter (Java) Theory | Curated by Someone | Medium

Other series on this account

• List: Cybersecurity | Curated by Someone | Medium
• List: Python Guide | Curated by Someone | Medium
• List: Python Guide - Theory | Curated by Someone | Medium
• List: GoLang Guide | Curated by Someone | Medium
• List: GoLang Guide Theoretical | Curated by Someone | Medium
• List: My Progress - Matplotlib | Curated by Someone | Medium

What’s Next

This program created 48 Thread objects by hand, and that is already awkward for what is a very small amount of work. In the next lesson we hand that job to the JVM with ExecutorService and thread pools, and then look at CompletableFuture for results that arrive later.

Read the companion theory article here.

Learning Java from Scratch with Arbiter: Threads, Race Conditions, synchronized, and Deadlocks was originally published in Dev Genius on Medium, where people are continuing the conversation by highlighting and responding to this story.

Breachlab Guide Theory • Day 2 | Why “options” are only a convention, why the shell splits your words before any program sees them, and how that gap turned into a real class of vulnerabilities

Disclaimer: This series is for educational purposes only. Everything discussed is studied on Breachlab’s legal practice targets and through published research. Don’t test these ideas on systems you don’t own or don’t have written permission to test. In line with Breachlab’s guidelines, this article explains concepts and history; it contains no level answers or step-by-step solutions.

In the practical guide for Ghost Level 1, a file that was plainly visible in ls refused to open. A leading dash made cat treat the name as an option, a lone dash made it read the keyboard, and a space split one filename into two.

It’s tempting to file that under “Linux quirks” and move on. But when you ask why it happens, you uncover one of the oldest and most persistent ideas in Unix security. A command line goes through two interpreters before anything is executed, and neither of them knows what the other intended.

This article accompanies the practical guide for Day 2 and focuses on the reasoning behind the behaviour rather than on typing commands. We’ll look at where “options” actually come from, what the shell does to your text before any program sees it, why Linux allows such strange filenames at all, and how security researchers turned this ambiguity into real attacks.

The Kernel Has No Idea What an Option Is

Why the dash is a social agreement between programs, not a feature of the operating system.

When the shell starts a program, it eventually calls execve, the system call that replaces a process with a new program. One of its parameters is argv: an array of plain strings. That's it. The kernel doesn't mark any string as "option" or "filename". -v, notes.txt and --help all arrive as identical, unlabelled pieces of text.

Deciding what each string means is entirely the job of the program receiving it. Early Unix programs each parsed their own arguments by hand, and a habit formed: arguments starting with - are switches, and everything else is something to work on. That habit was later formalised by the getopt library function and then written into the POSIX standard as the Utility Syntax Guidelines.

Three of those guidelines explain almost everything you saw on Level 1:

• Guideline 4 says all options should be preceded by the - delimiter character. This is the rule that turns -v into a switch.
• Guideline 10 says the first -- that isn't an option-argument marks the end of options. That's why cat -- -v works.
• Guideline 13 says a - operand should mean only standard input, or a file named -. The standard itself admits the ambiguity, and leaves it to each utility to choose.

Notice the language: should. These are guidelines, and every program implements them itself. That’s why -- works with cat but isn't guaranteed to work with every tool, and why the ./ prefix (which sidesteps option parsing entirely) is the more universal fix.

The Shell Splits Your Words Before Any Program Reads Them

Why a space is a delimiter until you say otherwise, and why quoting can’t save a filename that starts with a dash.

Before execve is ever called, the shell has to turn your single line of text into that argv array. The POSIX Shell Command Language defines this as a series of steps: tokenising the line, expanding variables and wildcards, field splitting on whitespace (controlled by the IFS variable), and finally quote removal.

That order explains the space problem precisely. An unquoted space is a word boundary, so field notes.txt becomes two array elements. Quotes and backslashes tell the shell "don't split here", so the name reaches the program as one string.

It also explains a subtle point from the guide: quoting doesn’t fix a leading dash. Quote removal is the last thing the shell does, so cat "-v" hands cat the exact same string as cat -v. The quotes protected the text from the shell, but the program never saw them, and it applies its own option rules regardless.

So there are two separate interpreters with two separate rulebooks. The shell decides where arguments begin and end, and the program decides what each argument means. Ghost Level 1 lives in the gap between them.

Filenames Can Be Almost Anything

Why Linux allows names that break its own tools.

You might wonder why Linux allows a file called - at all. The answer is that, at the filesystem level, a Unix filename is almost any sequence of bytes. Only two are forbidden: the / that separates path components, and the NUL byte that ends a string in C. Spaces, leading dashes, tabs, even newlines are all legal.

POSIX does define a portable filename character set (letters, digits, period, underscore and hyphen) and treats names starting with a hyphen as non-portable. But “non-portable” is advice, not enforcement.

The security researcher David A. Wheeler has written about this tension since 2009 in his essay Fixing Unix/Linux/POSIX Filenames. His argument is that this near-total freedom causes a steady stream of bugs and security flaws, because almost no script handles every legal name correctly. His recommendations include forbidding control characters, restricting leading dashes, and prefixing wildcards with ./. He also notes that the 2024 POSIX revision chose to keep allowing newlines in names and add tooling to cope with them instead, which he considers only a partial fix.

The lesson for a beginner is simple. The filesystem lets an attacker choose almost any name, and the tools you run assume names are well-behaved. That gap is where problems appear.

From Annoyance to Vulnerability: Argument Injection

How “my cat command is hanging” turns into “someone else’s command is running my options”.

On Level 1 the confusion only hurt you. Now flip it around: what if someone else controls the filename, and you (or a privileged script) run the command?

In 2014, Leon Juranic of DefenseCode published Back To The Future: Unix Wildcards Gone Wild. The core idea is elegant and a little alarming. An unquoted wildcard like * expands to every filename in a directory, and that expansion happens before the program parses its options. If an attacker can create files in a directory that another user, cron job or web application later processes with a wildcard, they can plant files whose names begin with -. The program then receives those names as options. The paper showed this affecting common tools such as rm, chown, chmod, tar and rsync, ranging from changed file ownership to, in some configurations, command execution.

Nothing in that attack is a memory-corruption bug. Every program behaves exactly as documented. The weakness is purely that data (a filename) is allowed to cross into the space reserved for instructions (options).

MITRE catalogues this pattern as CWE-88: Improper Neutralization of Argument Delimiters in a Command (“Argument Injection”). Software builds a command for another component, doesn’t properly separate its intended arguments, and so input gets read as extra options.

It isn’t only a problem in shell scripts, either. In 2017, CVE-2017–1000117 showed that Git could be handed an ssh:// URL whose host part began with a dash. Git passed that "hostname" to the local ssh client, which read it as a command-line option instead of a host. A crafted repository URL, including one hidden in a submodule, could therefore lead to command execution on the machine running Git. Patched versions added checks that reject hosts and paths beginning with a dash.

That’s the same principle you met on Level 1, just in a different program.

The Principle Behind Every Fix

Every defence is a way of removing ambiguity about which strings are data.

Looked at through theory, the four fixes from the practical guide aren’t four tricks. They’re three applications of one idea:

• Make data look unlike instructions. A ./ prefix means the argument can't start with a dash. Wheeler recommends exactly this for wildcards (./* instead of *).
• Mark where the instructions end. -- is the explicit delimiter from POSIX Guideline 10, and it's the standard mitigation for CWE-88 when a program supports it.
• Don’t hand the program a name at all. Input redirection (<) makes the shell open the file, so the program never parses a filename.

In application code, the same principle shows up as passing arguments as an array instead of building a shell string (for example, Python’s subprocess.run([...]) without shell=True), and as allowlisting which options a user can influence.

Why These Concepts Matter

Each idea in this article answers part of one question: how does a line of text become a decision about what to execute?

• execve and argv show that the operating system passes plain strings, and that "options" are a convention every program implements for itself.
• The POSIX Utility Syntax Guidelines explain why the dash means “option”, why -- ends options, and why a lone - means standard input.
• Shell word splitting and quote removal explain why spaces break names, and why quotes can’t rescue a leading dash.
• The near-anything filename rules, and Wheeler’s critique of them, explain why attackers get to pick the strings in the first place.
• Juranic’s wildcard research, CWE-88 and the 2017 Git CVE show how the same ambiguity turned into real security flaws.

For a pentester, this is a mindset as much as a fact. Whenever a system takes input and later hands it to another program, ask: could this data be read as an instruction? That question will come up again throughout Breachlab, from shell scripts to web apps to CI/CD pipelines.

The Breachlab Journey Continues

This series works through Breachlab one level at a time, pairing every practical guide with a theory companion like this one. Ghost builds the Linux foundations; later tracks build on them for post-exploitation, web, cloud and Active Directory work.

Breachlab is free and runs entirely on donations. If it’s teaching you something, you can support it here: breachlab.org/donate

Read the practical companion to this article: Learning Cybersecurity from Scratch with Breachlab — Ghost Level 1

Understand the concept in detail

Watch the deep dive

Follow along with the complete Breachlab playlist on YouTube: Breachlab playlist on Assert True. The channel is Assert True on YouTube.

GitHub Repository

My notes and progress for every level live here: github.com/Someone-anon-coder/Breachlab

Follow the complete series

• Breachlab Ghost I list on Medium: every Breachlab guide and theory article in one place
• Cybersecurity list on Medium: my wider cybersecurity learning notes

My other learning series

Arbiter (Java) Guide · Arbiter (Java) Theory · Python Guide · Python Guide Theory · GoLang Guide · GoLang Guide Theory · My Progress — Matplotlib

What’s Next?

In the next Breachlab Guide Theory article, we’ll explore the theory behind Ghost Level 2: why Unix hides “dotfiles” by default (a convention that famously began as a shortcut in the code of ls), what that means for where configuration and secrets end up, and why "not shown" is never the same as "protected".

Happy hacking! 🛡️

References

1. The Open Group, Base Specifications Issue 7, Chapter 12: Utility Conventions (Utility Syntax Guidelines): pubs.opengroup.org/onlinepubs/9699919799.2008edition/basedefs/V1_chap12.html
2. The Open Group, Shell Command Language: Word Expansions and Quoting: pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html
3. GNU Bash Reference Manual, Word Splitting: gnu.org/software/bash/manual/html_node/Word-Splitting.html
4. David A. Wheeler, Fixing Unix/Linux/POSIX Filenames (2009, updated 2024): dwheeler.com/essays/fixing-unix-linux-filenames
5. David A. Wheeler, Filenames and Pathnames in Shell: How to do it Correctly: dwheeler.com/essays/filenames-in-shell.html
6. Leon Juranic (DefenseCode), Back To The Future: Unix Wildcards Gone Wild (2014): exploit-db.com/papers/33930
7. Help Net Security, Exploiting wildcards on Linux (2014): helpnetsecurity.com/2014/06/27/exploiting-wildcards-on-linux
8. MITRE, CWE-88: Improper Neutralization of Argument Delimiters in a Command (“Argument Injection”): cwe.mitre.org/data/definitions/88.html
9. NSFOCUS, Git SSH CVE-2017–1000117 analysis: blog.nsfocus.net/git-ssh-cve-2017–1000117

Understanding the Linux Command Line — The Theory Behind Options, Word Splitting, and How a… was originally published in CodeX on Medium, where people are continuing the conversation by highlighting and responding to this story.

Day 2 of my Breachlab Learning Journey | Why a filename that starts with a dash or contains a space breaks a perfectly normal command, and four ways to tell the shell exactly what you mean

Disclaimer: This series is for educational purposes only. Everything here happens on Breachlab’s own practice targets, which exist to be attacked legally. Don’t try these techniques on systems you don’t own or don’t have written permission to test. Following Breachlab’s guidelines, I won’t share passwords, flags or a copy-paste solution. Every example below runs on a small practice folder you build yourself, so you learn the technique and still solve the real level on your own.

You log in. You run ls. The file you need is sitting right there, plainly listed.

You type cat and the filename, hit Enter, and... nothing. The cursor just blinks at you. Or you get an error saying the file doesn't exist, about a file you can literally see.

That was my first ten minutes on Ghost Level 1, and honestly it was a more useful lesson than any of the “find the hidden thing” levels. Nothing is hidden here. Nothing is locked. The only thing standing between you and the password is how your shell reads what you typed.

This is Day 2 of my Breachlab Learning Journey, where I’m working through Breachlab’s tracks level by level and documenting what each one actually teaches. On Day 1 (Ghost Level 0), the job was getting connected over SSH and learning to look around a box before touching anything. Today builds directly on that: you know how to find files, and now you find out that finding a file and naming it correctly are two different skills.

Today’s Topics

• Why some filenames get mistaken for command options
• Why a space in a filename quietly turns one argument into two
• Four reliable ways to refer to an awkward file
• A habit for spotting trouble before you hit Enter

First, Build a Practice Folder

I’m not going to walk you through the real level, so let’s build something with the same kinds of problems. On any Linux machine or VM, run this:

mkdir -p ~/lab/ghost1 && cd ~/lab/ghost1

echo "practice value one"   > ./-
echo "practice value two"   > ./-v
echo "practice value three" > "./field notes.txt"
ls -la

You now have three files whose names are deliberately annoying: one called just -, one called -v, and one with a space in it. The real level uses its own filename and its own twist, so treat these as training weights, not the answer.

Problem 1: The Filename That Looks Like an Option

Try reading the second file the obvious way:

cat -v

Your terminal just sits there. It hasn’t frozen; cat is waiting for you to type something. Press Ctrl + C to get out.

Here’s what happened. Almost every Linux command treats an argument starting with - as an option, a switch that changes how the command behaves. -v happens to be a real option for cat ("show non-printing characters"). So cat read your filename, decided it was a setting, found no actual file to read, and fell back to reading from your keyboard.

The lone - file trips you up in a similar way. By long-standing Unix convention, a single - on its own means "standard input": read from the keyboard or a pipe instead of a file. So cat - doesn't open the file named - either. It waits for you to type.

The file isn’t missing. The command just never looked for it, because it understood your filename as an instruction.

Problem 2: The Filename With a Space

Now the third file:

cat field notes.txt

This time you get two errors, one for field and one for notes.txt. Before cat even runs, your shell splits what you typed on spaces to decide where each argument starts and ends. So cat was handed two separate names, and neither exists.

The space is perfectly legal in a filename. Linux allows almost any character in one, but the shell has its own rules about what a space means on a command line, and those rules win unless you override them.

Four Ways to Tell the Shell What You Mean

Every one of these fixes does the same thing in a different way: it removes the ambiguity, so the command can only read your argument as a filename.

1. Give it a path

cat ./-v
cat ./-

./ means "in the current directory." The argument now starts with a dot, not a dash, so no command will ever mistake it for an option. This is the fix I reach for first because it works everywhere, even with older or unusual tools.

2. Use the double dash

cat -- -v

A bare -- is a widely followed convention that means "options end here; everything after this is a filename." Most standard Linux tools honor it. It's very handy in scripts, but not every program supports it, so test it before you rely on it.

One catch: cat -- - still reads from standard input, because many tools treat a lone - as "stdin" even after --. For that particular name, the path fix is the reliable one.

3. Quote or escape it

cat "field notes.txt"
cat field\ notes.txt

Quotes tell the shell “this whole thing is one argument.” A backslash does the same for a single character, so \ means "a literal space, don't split here." Quoting fixes the space problem, but not the dash problem: cat "-v" still gets read as an option, because the quotes are removed before cat ever sees the text.

4. Redirect it in

cat < -

With <, the shell opens the file and feeds its contents to the command. The command never sees a filename at all, so it has nothing to misread. It's a neat trick, and it's useful to understand even when you don't need it.

A Habit Worth Building: Look Before You Type

Most of the pain on this level comes from assuming a filename is “normal.” Two small checks help you catch the weird ones before you hit Enter:

ls -Q        # wraps every name in quotes, so spaces and odd characters stand out
ls -b        # shows escapes for special characters, like field\ notes.txt
file ./*     # tells you what each file actually contains

And use Tab completion. Type the first character or two and press Tab; the shell fills in the name and adds whatever escaping it needs. Half the time, the shell is quietly showing you the answer to "how do I type this name safely?"

When you try the actual level, slow down at the ls step. Look at exactly what the filename is, work out which problem it causes (option? standard input? splitting? more than one?), then pick the fix that removes that ambiguity. That's the entire skill, and it's yours to apply.

Key Takeaway

A command line isn’t just text. It’s text the shell splits into arguments, which the command then sorts into options and files. Ghost Level 1 lives in the gap between those two steps. A leading dash makes a filename look like an option, a lone dash means standard input, and a space splits one name into two. Every fix (a ./ path, --, quotes or escapes, or < redirection) works by taking away the command's room to guess.

This matters well beyond one wargame level. When attacker-controlled filenames reach scripts that don’t handle them carefully, this exact confusion stops being a nuisance and becomes a vulnerability. I dig into that history, and the standards that explain why Linux behaves like this, in today’s theory companion:

Understanding Linux Command-Line Parsing — The Theory Behind Ghost Level 1

The Breachlab Journey Continues

One level at a time, this series works through Breachlab’s tracks, from Ghost’s Linux fundamentals toward post-exploitation, web, cloud and Active Directory. Each level gets a practical guide like this one and a theory companion explaining the “why.”

Breachlab is free and runs entirely on donations. If it’s teaching you something, you can support it here: breachlab.org/donate

Watch the concept in detail

Watch the deep dive

Follow along with the complete Breachlab playlist on YouTube: Breachlab playlist on Assert True. The channel is Assert True on YouTube.

GitHub Repository

My notes and progress for every level live here: github.com/Someone-anon-coder/Breachlab

Follow the complete series

• Breachlab Ghost I list on Medium: every Breachlab guide and theory article in one place
• Cybersecurity list on Medium: my wider cybersecurity learning notes

My other learning series

Arbiter (Java) Guide · Arbiter (Java) Theory · Python Guide · Python Guide Theory · GoLang Guide · GoLang Guide Theory · My Progress — Matplotlib

What’s Next?

On Day 3, we move to Ghost Level 2: files that don’t show up in a normal listing at all, why Linux hides some names by default, and how to stop trusting the first ls you run.

Happy hacking! 🛡️

Learning Cybersecurity from Scratch with Breachlab — Ghost Level 1: The File You Can’t Open by Its… was originally published in InfoSec Write-ups on Medium, where people are continuing the conversation by highlighting and responding to this story.

This article was prepared with AI assistance. The companion video’s narration is synthesized speech and the presenter is an animated illustration, not a real person. This guide teaches methodology and intentionally does not reveal wargame solutions or passwords.

LAB ENVIRONMENT — commands executed in a local throwaway sandbox

# Throwaway reproduction sandbox (runs locally as unprivileged user; display_user="dev")
mkdir -p ~/lab/{reports,scripts,.config}
cd ~/lab
echo "README content" > README.txt
echo "status ok" > reports/q3.txt
echo "API_TOKEN=example-not-a-real-token" > deploy.env
chmod 640 deploy.env
echo "rotate the deploy token before friday" > .reminder
chmod 600 .reminder
printf "todo: archive old reports\r\ntodo: rotate keys\r\n" > notes.txt
printf "\x00\x01\x02\xfe\xff\x00\x00\x00CONNECT db.internal:5432\x00\xde\xad\xbe\xef\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00user=deploy\x00" > data.bin
chmod 755 scripts

What this level is

You land on a shell on a machine you have never seen. There is no map, no architecture diagram, and no administrator telling you what matters. What you do in the next sixty seconds determines whether the rest of the session is disciplined, repeatable, and effective or chaotic and noisy. Professional operators on both offensive engagements and incident response investigations begin with the same three fundamental questions: who am I, where am I, and what is here?

In this guide, we answer those questions against BreachLab’s Ghost track, Level 0, titled “First Contact”. The level page lists three foundational commands: ls, cd, and cat.

Behind those three simple commands lie decades of computer science history and operating system architecture. In this guide, we examine:

1. Why identity in Unix is governed by kernel-level numeric IDs rather than environment variables.
2. How the filesystem hierarchy and hidden dot-files originated from a historical shortcut.
3. How the access matrix compresses into nine permission bits, and how discretionary access control creates predictable security vulnerabilities.

• Platform: BreachLab, a free, legal hands-on platform where every level is an isolated, real Linux environment.
• Track & Level: Ghost I · Level 0 — First Contact (100 points).
• Target Connection: ssh ghost0@204.168.229.209 -p 2222
• Commands Covered: whoami, pwd, ls, cd, cat, file, head, tail, strings, xxd, stat, type.
• Level Objective: Locate the credentials for the next account (ghost1).

Watch the full long-form video walkthrough:

Follow along with the complete series playlist:

Compliance Note: In strict adherence to BreachLab creator rules, this article teaches the underlying method, mechanics, and security theory. No passwords, flags, or copy-paste solutions are published.

Today’s Topics

• The Orientation Loop: Executing the initial sixty-second triage using whoami, pwd, and ls -la.
• SSH Protocol Mechanics: Understanding TCP handshakes, Diffie-Hellman key agreement, and host verification.
• Identity and Complete Mediation: Why the kernel evaluates effective UIDs rather than shell variables.
• Filesystem Enumeration: Walking directory tables, understanding inodes, and unmasking dot-files.
• Permission Bit Triads: Decoding octal modes, directory traversal mechanics, and access matrix theory.
• Path Traversal with Shell Builtins: Why cd must execute inside the parent shell process.
• Defensive Content Inspection: Safely examining unknown streams using file, cat -A, and strings.
• Unsecured Credentials in Files (T1552.001): Real-world impact, kill chain positioning, and honeytoken detection.

Orienting on an Unknown Target: whoami and pwd

When I first connect to a remote target, before typing any complex commands or running automated scripts, I establish orientation. I need to know the security context of the process running my shell and my exact position in the directory hierarchy.

I begin by asking the kernel for my active identity. I run whoami:

$ whoami
dev

The command returns dev (or ghost0 when logged into the live target). Behind this single word, the whoami utility executed the geteuid() system call to fetch the numeric effective user ID from the operating system kernel, then performed a lookup against /etc/passwd.

To illustrate why this matters, consider what happens if I rely on the common $USER environment variable instead:

$ echo $USER
dev
$ USER=root
$ echo $USER
root
$ whoami
dev

The environment variable $USER is completely under user-space control. An adversary or a misconfigured script can alter it at will. The kernel's reference monitor completely ignores $USER when enforcing access permissions; it consults only the numeric UID bound to the process structure in memory.

Next, I check my location in the filesystem with pwd:

$ pwd
/home/dev/lab

The pwd (print working directory) command prints the absolute path of the current directory. This path serves as the anchor point for every relative path resolution performed by the kernel during system calls like open() and execve().

Takeaway: Always query the kernel directly for process identity rather than trusting environment variables that user space can manipulate.

Anatomy of an SSH Connection

To reach Level 0, we initiate an SSH connection to port 2222:

$ ssh ghost0@204.168.229.209 -p 2222

This command initiates a multi-stage protocol exchange:

1. Transport Layer Handshake: The client opens a TCP connection to port 2222. Port 2222 is chosen by convention for secondary or containerised SSH daemons, avoiding privileged port binding (ports below 1024 require root privileges).
2. Version Exchange: Both client and server exchange identification strings (e.g., SSH-2.0-OpenSSH_8.9p1).
3. Key Exchange (KEX): Both sides agree on cryptographic algorithms and execute a Diffie-Hellman or elliptic curve exchange to establish a shared session key without transmitting it over the wire.
4. Server Authentication: The server signs the exchange with its private host key. The client verifies this signature against its local ~/.ssh/known_hosts file.
5. User Authentication: Once encryption is established, the client sends user authentication credentials (in Level 0, password authentication; in hardened environments, Ed25519 public keys or certificates).

Takeaway: SSH ensures confidentiality, integrity, and server authentication before a single credential character is sent across the network.

Enumerating the Filesystem: ls and What Else You Could Use

With identity and working directory confirmed, the next priority is inventorying what is present.

I start with a plain ls:

$ ls
README.txt
data.bin
deploy.env
notes.txt
reports
scripts

A standard ls reads the directory data structure and outputs filenames alphabetically. However, a plain listing hides crucial information. In Unix, files whose names begin with a period (.) are excluded from default directory listings.

To see all entries, I supply the -a flag:

$ ls -a
.
..
.config
.reminder
README.txt
data.bin
deploy.env
notes.txt
reports
scripts

Notice what appeared: . (the current directory), .. (the parent directory), .config (a configuration directory), and .reminder (a hidden text file).

In security assessments, dot-files are prime locations for misconfigured credentials, shell histories (.bash_history), cloud tokens (.aws/), and private keys (.ssh/id_rsa).

To inspect file metadata, I combine flags into the standard enumeration command ls -la:

$ ls -la
total 40
drwxrwxr-x 5 dev dev 4096 Oct  7 07:02 .
drwxrwxr-x 3 dev dev 4096 Oct  7 07:02 ..
drwxr-xr-x 2 dev dev 4096 Oct  7 07:02 .config
-rw------- 1 dev dev   38 Oct  4 07:02 .reminder
-rw-r--r-- 1 dev dev   80 Oct  7 07:02 README.txt
-rw-r--r-- 1 dev dev   61 Sep  7 07:02 data.bin
-rw-r----- 1 dev dev   67 Oct  2 07:02 deploy.env
-rw-r--r-- 1 dev dev   46 Sep 28 07:02 notes.txt
drwxr-xr-x 2 dev dev 4096 Oct  7 07:02 reports
drwxr-xr-x 2 dev dev 4096 Oct  7 07:02 scripts

What if ls is not available?

In minimal Docker containers or restricted shells, the ls binary may be absent or restricted. Because filename expansion (globbing) is handled directly by the shell before command execution, we can enumerate files using built-in shell features:

$ echo *
README.txt data.bin deploy.env notes.txt reports scripts

When recursive enumeration is needed, find and tree provide structured visibility:

$ tree -a -L 2
.
├── .config
│   └── app.conf
├── .reminder
├── README.txt
├── data.bin
├── deploy.env
├── notes.txt
├── reports
│   ├── q3.txt
│   └── q4.txt
└── scripts
    └── backup.sh

4 directories, 9 files

Takeaway: Plain ls conceals administrative configuration; standard enumeration requires ls -la or native shell globbing.

Decoding Access Rights: Reading a Permission String

The output of ls -la presents a 10-character string at the beginning of each line. Understanding this string is essential for recognizing privilege boundaries.

Consider the line for deploy.env: -rw-r----- 1 dev dev 67 Oct 2 07:02 deploy.env

We dissect the string from left to right:

• Character 1 (File Type): - denotes a regular file. A d indicates a directory, while l denotes a symbolic link.
• Characters 2–4 (Owner Permissions): rw- indicates the owner (dev) has read (r=4) and write (w=2) permissions, but no execute (x=1). Total = 6.
• Characters 5–7 (Group Permissions): r-- indicates members of the owning group (dev) have read permission only. Total = 4.
• Characters 8–10 (Others Permissions): --- indicates any user who is neither the owner nor a member of the group has zero permissions. Total = 0.

The combined octal mode is 640.

We can verify this numeric representation directly using stat:

$ stat -c '%A  %a  %U:%G  %s bytes  %n' README.txt .reminder scripts/backup.sh deploy.env
-rw-r--r--  644  dev:dev  80 bytes  README.txt
-rw-------  600  dev:dev  38 bytes  .reminder
-rwxr-xr-x  755  dev:dev  40 bytes  scripts/backup.sh
-rw-r-----  640  dev:dev  67 bytes  deploy.env

The Crucial Difference for Directories

For files, r allows reading content, w allows modifying bytes, and x allows running the file as an executable program.

For directories, the bits have distinct meanings:

• r (Read): Grants the ability to list the names of files inside the directory (e.g., with ls).
• w (Write): Grants the ability to create, delete, or rename files within the directory (contingent on execute permissions).
• x (Execute/Search): Grants the ability to enter the directory (cd) and access the inodes of files inside it.

Takeaway: Without directory execute permissions, read permissions allow listing names but prevent accessing file contents or metadata.

Navigating the Hierarchy: Moving Around with cd

Navigating the directory tree is accomplished using cd (change directory).

When we check the binary location of cd, we encounter a fundamental architectural distinction:

$ type cd ls cat pwd
cd is a shell builtin
ls is /usr/bin/ls
cat is /usr/bin/cat
pwd is a shell builtin

ls and cat are independent executable binaries located in /usr/bin/. When invoked, the shell calls fork() to create a child process, followed by execve() to load the binary into memory.

cd, however, is a shell builtin. In Unix process architecture, a child process inherits a copy of its parent's working directory, but any modifications made by the child to its own working directory do not propagate back to the parent. If cd were an external executable binary, running /usr/bin/cd reports would change the directory of the child process, which would then immediately exit, leaving the user's interactive shell exactly where it started.

Common Navigation Patterns

We test navigation shortcuts across our lab hierarchy:

$ cd reports && pwd && cd .. && pwd && cd ~ && pwd && cd - && pwd
/home/dev/lab/reports
/home/dev/lab
/home/dev
/home/dev/lab
/home/dev/lab

Key shortcuts:

• ..: References the parent directory entry.
• ~: Expands to the current user's home directory ($HOME).
• -: Returns to the previous working directory ($OLDPWD).

Takeaway: cd must be implemented inside the shell process because child processes cannot alter parent process state.

Inspecting Files Safely: cat and the Tools Around It

Once a target file is discovered, we must inspect its contents. The standard utility is cat (short for concatenate).

cat reads file streams sequentially and writes them to standard output. While effective for short text files, dumping arbitrary files directly with cat introduces risks:

1. Long files scroll off the screen, obscuring terminal history.
2. Binary files send control characters that can corrupt terminal settings or reprogram keyboard mappings.

To avoid this, we follow a two-step rule: identify first, read second.

We inspect file types using file:

$ file *
README.txt: ASCII text
data.bin:   data
deploy.env: ASCII text
notes.txt:  ASCII text, with CRLF line terminators
reports:    directory
scripts:    directory

Notice notes.txt: it is flagged as containing CRLF line terminators (Windows-style \r\n).

If we view notes.txt with plain cat, the carriage return is invisible. But when we pass -A (show all invisible characters), the hidden control character becomes evident:

$ cat -A notes.txt
todo: archive old reports^M$
todo: rotate keys^M$

The ^M represents carriage return (\r, ASCII 0x0D), while $ marks the newline (\n, ASCII 0x0A). If an automated script parses this file to extract a password or hostname, the carriage return remains attached to the string, causing subsequent authentication requests to fail silently.

Specialized Inspection Tools

When dealing with structured, binary, or large datasets, we use tailored utilities:

1. Inspecting Binaries with strings Instead of dumping raw bytes from data.bin, strings extracts contiguous printable ASCII sequences (minimum length 4 by default):

$ strings data.bin
CONNECT db.internal:5432
user=deploy

2. Exact Byte Inspection with xxd To view hexadecimal representations and offsets:

$ xxd data.bin
00000000: 0001 02fe ff00 0000 434f 4e4e 4543 5420  ........CONNECT 
00000010: 6462 2e69 6e74 6572 6e61 6c3a 3534 3332  db.internal:5432
00000020: 00de adbe ef00 0000 0000 0000 0000 0000  ................
00000030: 0075 7365 723d 6465 706c 6f79 00         .user=deploy.

3. Bounded Inspection with head and tail To read headers or trailing log entries without flooding standard output:

$ head -n 3 reports/q3.txt
line 001: status ok
line 002: status ok
line 003: status ok

4. Recursive Content Searching with grep When searching across an entire hierarchy for credentials:

$ grep -rn -i 'token' . 2>/dev/null
./.reminder:1:rotate the deploy token before friday
./deploy.env:2:API_TOKEN=example-not-a-real-token

Takeaway: Always classify unknown files with file before reading, and use cat -A to catch invisible carriage returns that break scripts.

Solving Level 0 on the Target Machine

Now we apply our orientation methodology to the live BreachLab Ghost 0 target.

We log in over SSH as ghost0. The session opens in /home/ghost0. We execute our triage loop:

$ whoami
ghost0
$ pwd
/home/ghost0
$ ls
workspace

The directory contains an entry named workspace. We move into it:

$ cd workspace
$ ls
archive

Inside workspace is a directory named archive. In enterprise systems, archive directories are frequently overlooked during credential rotation sweeps. We descend into archive:

$ cd archive
$ ls
credentials

We find a file named credentials. Before reading it, we verify its format:

$ file credentials
credentials: ASCII text

Finally, we display the credential:

$ cat credentials

(The password output is intentionally redacted in accordance with BreachLab platform rules).

The password string displayed in this file represents the flag for Level 0 and serves as the SSH authentication password for ghost1. We immediately log this credential in an external secure password manager before proceeding to Level 1.

Takeaway: Level 0 requires zero exploit payloads; disciplined filesystem traversal and directory enumeration locate the credential directly.

Running It

Here is the complete orientation session as executed in our sandbox environment, demonstrating the unbroken workflow from initial connection to secret identification:

$ whoami
dev
$ pwd
/home/dev/lab
$ ls -la
total 40
drwxrwxr-x 5 dev dev 4096 Oct  7 07:02 .
drwxrwxr-x 3 dev dev 4096 Oct  7 07:02 ..
drwxr-xr-x 2 dev dev 4096 Oct  7 07:02 .config
-rw------- 1 dev dev   38 Oct  4 07:02 .reminder
-rw-r--r-- 1 dev dev   80 Oct  7 07:02 README.txt
-rw-r--r-- 1 dev dev   61 Sep  7 07:02 data.bin
-rw-r----- 1 dev dev   67 Oct  2 07:02 deploy.env
-rw-r--r-- 1 dev dev   46 Sep 28 07:02 notes.txt
drwxr-xr-x 2 dev dev 4096 Oct  7 07:02 reports
drwxr-xr-x 2 dev dev 4096 Oct  7 07:02 scripts
$ stat -c '%A  %a  %U:%G  %s bytes  %n' deploy.env .reminder
-rw-r-----  640  dev:dev  67 bytes  deploy.env
-rw-------  600  dev:dev  38 bytes  .reminder
$ file *
README.txt: ASCII text
data.bin:   data
deploy.env: ASCII text
notes.txt:  ASCII text, with CRLF line terminators
reports:    directory
scripts:    directory
$ cat -A notes.txt
todo: archive old reports^M$
todo: rotate keys^M$
$ grep -rn -i 'token' . 2>/dev/null
./.reminder:1:rotate the deploy token before friday
./deploy.env:2:API_TOKEN=example-not-a-real-token

Reading it in groups:

• Identity & Context: whoami and pwd verify unprivileged user execution inside the lab root.
• Surface Discovery: ls -la unmasks hidden entries (.config, .reminder) that default listings omit.
• Permission Assessment: stat reveals that .reminder is restricted to mode 600 (owner read/write only), while deploy.env is mode 640.
• Stream Verification: file alerts us to Windows-style CRLF terminators in notes.txt, confirmed via cat -A.
• Targeted Extraction: grep recursively isolates credential assignments without manual file crawling.

Theory: Why Unix Looks the Way It Does

Fifty years of hierarchical abstraction and the accident of dot-files.

Sources

• Ritchie & Thompson, The UNIX Time-Sharing System (Communications of the ACM, 1974)
• Unix Programmer’s Manual, 1st Edition (Bell Laboratories, 1971)
• Filesystem Hierarchy Standard (FHS 3.0, Linux Foundation, 2015)

In 1969 at Bell Laboratories, Ken Thompson and Dennis Ritchie began designing a streamlined operating system on a discarded Digital Equipment Corporation PDP-7. The design principles they established remain the structural foundation of modern Linux systems.

Ritchie and Thompson formulated four enduring architectural tenets:

1. Hierarchical Namespace: A single rooted tree where directories contain file descriptors and pointers to subordinate directories.
2. Unstructured Byte Streams: A file is simply an arbitrary sequence of bytes. The operating system imposes no record delimiters or indexed access methods; interpretation is delegated entirely to user space.
3. Pervasive File Abstraction: Peripheral devices, storage disks, terminals (/dev/tty), and inter-process communication channels are presented as file descriptors within the namespace.
4. Tool Composition: Modular utilities that perform a single task well, linked via unidirectional pipes (introduced by Douglas McIlroy in 1973).

In November 1971, the Unix Programmer’s Manual (First Edition) formally documented ls, cat, and chdir (the predecessor to cd). In the 1974 CACM paper, directories were defined as simple lookup tables mapping character string names to numeric internal indices called i-numbers (inodes).

The Accident of Dot-Files

The convention of “hidden” files originated as a programming shortcut rather than a deliberate security mechanism. According to a widely cited historical account by Rob Pike, when Thompson and Ritchie added directory navigation, every directory received two structural entries: . (pointing to the directory itself) and .. (pointing to its parent). To prevent these two entries from cluttering directory listings, a check was added to ls: if the first character of a filename was ., the entry was skipped.

This simple prefix check inadvertently caused any file beginning with a period to be hidden from the default listing. Developers quickly adopted this behavior to stash user configuration files (.profile, .bashrc) out of everyday view. In modern security testing, this design artifact makes dot-files the primary repository for unmonitored credentials.

Under the Hood: Directory Inodes

In modern Linux filesystems (ext4, XFS), a directory is an inode containing data blocks filled with directory entries (struct linux_dirent64):

$ ls -i
5771093 README.txt
5771103 data.bin
5771104 deploy.env
5771095 notes.txt
5771100 reports
5771098 scripts

The filename is merely a human-readable link pointing to an inode number. The permissions, file size, timestamps, and data block pointers reside entirely inside the inode table, not within the directory entry itself.

Theory: Why SSH Exists (History, Diffie-Hellman, and MITM)

From plaintext cleartext telnet to cryptographically authenticated sessions.

Sources

• Ylönen, SSH — Secure Shell Remote Login (1995)
• RFC 4251: The Secure Shell (SSH) Protocol Architecture (IETF, 2006)
• Diffie & Hellman, New Directions in Cryptography (IEEE Transactions on Information Theory, 1976)

Before SSH, remote administration relied on Telnet (RFC 854) and the Berkeley r-commands (rlogin, rsh). These protocols transmitted every packet across local networks and the ARPANET in plaintext. Anyone with network access could capture passwords and keystrokes using packet sniffers.

In 1995, following a major password sniffing incident at the Helsinki University of Technology, Tatu Ylönen designed SSH-1 as a secure replacement. In 1999, the OpenBSD project developed OpenSSH, which became the global standard implementation, formalized as SSH-2 in RFCs 4251–4254 in 2006.

Cryptographic Foundation: The Diffie-Hellman Key Agreement

The central challenge of remote login across untrusted networks is key distribution: how can two machines agree on a shared secret without an eavesdropper deducing it?

Whitfield Diffie and Martin Hellman solved this in 1976 using modular arithmetic over a cyclic group. Here is a worked example using deliberately small numbers:

1. Client and server publicly agree on a generator base g=5g=5 and a prime modulus p=23p=23.
2. The client picks a private secret a=6a=6 and computes the public value A=ga(modp)=56(mod23)=15625(mod23)=8A=ga(modp)=56(mod23)=15625(mod23)=8. The client transmits A=8A=8.
3. The server picks a private secret b=15b=15 and computes the public value B=gb(modp)=515(mod23)=19B=gb(modp)=515(mod23)=19. The server transmits B=19B=19.
4. The client computes the shared secret: K=Ba(modp)=196(mod23)=47045881(mod23)=2K=Ba(modp)=196(mod23)=47045881(mod23)=2.
5. The server computes the shared secret: K=Ab(modp)=815(mod23)=35184372088832(mod23)=2K=Ab(modp)=815(mod23)=35184372088832(mod23)=2.

Both sides obtain the shared secret K=2K=2. An adversary observing the network sees g=5g=5, p=23p=23, A=8A=8, and B=19B=19. Computing aa or bb requires solving the discrete logarithm problem (gx≡A(modp)gx≡A(modp)), which is computationally infeasible when pp is a 2048-bit prime or an elliptic curve (Curve25519).

The Man-in-the-Middle (MITM) Threat

Diffie-Hellman establishes encryption against passive wiretapping, but it is vulnerable to an active attacker who intercepts communications and negotiates independent keys with each side.

To prevent this, the SSH server signs the key exchange using its asymmetric host key (RSA, ECDSA, or Ed25519). The client verifies this signature against its local known_hosts file. On the very first connection, the client must rely on Trust On First Use (TOFU), displaying the key fingerprint for manual user confirmation.

Theory: Identity and the Reference Monitor

Complete mediation, separation of identity, and the kernel boundary.

Sources

• Anderson, Computer Security Technology Planning Study (US Air Force, ESD-TR-73–51, 1972)
• Saltzer & Schroeder, The Protection of Information in Computer Systems (Proceedings of the IEEE, 1975)

In 1972, James P. Anderson introduced the foundational concept of the Reference Monitor: an abstract machine that mediates all access attempts by active subjects to passive objects.

Figure 13: Anderson’s Reference Monitor concept: mediating process requests against object security policies.

Anderson defined three mandatory properties for any reference monitor:

1. Complete Mediation: The monitor must be invoked on every single access attempt without exception.
2. Tamper Resistance: The monitor mechanism itself must be protected from unauthorized modification.
3. Verifiability: The monitor must be small and simple enough to be formally analyzed and proven correct.

In the Linux kernel, the Virtual Filesystem (VFS) and the Linux Security Module (LSM) hooks act as the reference monitor. When a process issues an open() or unlink() system call, the kernel intercepts execution, evaluates the credentials stored in the calling process's struct cred, and compares them with the target inode's mode and ownership fields.

The AAA Framework

Identity is partitioned across three distinct security operations:

• Authentication: Verifying who the entity is (e.g., SSH password verification).
• Authorization: Determining what the authenticated entity is permitted to do (e.g., filesystem permission bit checks).
• Accounting (Auditing): Recording what the entity actually did (e.g., Linux Audit logs via auditd).

Real vs. Effective UIDs

Every Linux process maintains multiple UID attributes:

• Real UID (RUID): Identifies the actual user who spawned the process.
• Effective UID (EUID): Evaluated by the kernel for filesystem and IPC permission checks.
• Saved UID (SUID): Enables setuid binaries to drop and regain elevated privileges safely.

When an unprivileged user executes a command, RUID and EUID match. In subsequent Ghost levels, understanding the divergence between RUID and EUID becomes the cornerstone of privilege escalation analysis.

Theory: Enumeration as a Method (Kill Chain, ATT&CK, NIST)

Systematic discovery versus aimless exploration.

Sources

• Hutchins, Cloppert & Amin, Intelligence-Driven Computer Network Defense Informed by Analysis of Adversary Campaigns and Intrusion Kill Chains (Lockheed Martin, 2011)
• MITRE ATT&CK Framework: Discovery Tactic (TA0007) (2015–Present)
• NIST Special Publication 800–115, Technical Guide to Information Security Testing and Assessment (2008)

Security assessments and forensic investigations do not proceed through random trial and error; they operate within structured threat models.

1. The Intrusion Kill Chain

Published in 2011 by Lockheed Martin researchers, the Cyber Kill Chain defines the linear phases of a cyberattack: Reconnaissance → Weaponization → Delivery → Exploitation → Installation → Command & Control (C2) → Actions on Objectives.

When an attacker establishes an interactive shell on a target, the engagement has passed the Exploitation phase. The attacker enters the initial phase of Actions on Objectives, where local internal discovery is required to understand the operating environment.

2. MITRE ATT&CK Mapping

The MITRE ATT&CK matrix decomposes adversary behavior into specific tactics and techniques. The actions in Level 0 map directly to the Discovery (TA0007) tactic:

• T1033 (System Owner/User Discovery): Running whoami or id to identify the active user context.
• T1083 (File and Directory Discovery): Executing ls, find, or tree to map directory trees and locate sensitive files.
• T1552.001 (Credentials in Files): Accessing unencrypted configuration files, archives, and flat files to harvest credentials.

3. NIST SP 800–115 Methodology

NIST outlines four phases of penetration testing: Planning, Discovery, Attack, and Reporting. Discovery is the engine that drives exploitation: each command run on a target trades operational risk (generating audit logs) for information gain.

Attacker vs. Defender Trade-offs: Quiet vs. Loud Enumeration

• Quiet Enumeration: Running native shell builtins and a minimal set of standard utilities (whoami, pwd, ls -la). This generates minimal telemetry and blends into normal administrative traffic.
• Loud Enumeration: Running automated post-exploitation scripts (such as LinPEAS or automated find loops) that touch thousands of files in seconds. This provides rapid discovery but triggers volume-based detection rules in modern Endpoint Detection and Response (EDR) sensors.

Theory: Access Control (The Matrix, DAC vs. MAC, and Design Principles)

Lampson’s access matrix, the POSIX compression, and the Saltzer-Schroeder axioms.

Sources

• Lampson, Protection (ACM Operating Systems Review, 1974 / 5th Princeton Conference, 1971)
• Saltzer & Schroeder, The Protection of Information in Computer Systems (Proceedings of the IEEE, 1975)
• Department of Defense Trusted Computer System Evaluation Criteria (TCSEC / “Orange Book”, DoD 5200.28-STD, 1983)

In 1971, Butler Lampson formalized access control theory using the Access Matrix Model.

In Lampson’s model:

• The rows represent Subjects (active processes executing on behalf of users).
• The columns represent Objects (passive entities: files, directories, sockets).
• Each cell M[S,O]M[S,O] defines the exact access rights held by subject SS over object OO.

Real-world operating systems cannot store an uncompressed matrix with thousands of users and millions of files. Systems store the matrix in one of two ways:

1. By Row (Capabilities): Each process holds a ticket or capability list granting specific operations.
2. By Column (Access Control Lists — ACLs): Each object stores a list of authorized subjects and their permitted actions.

The Unix Compression

Unix implements a compressed column-based ACL. Instead of listing every user individually, each file inode stores exactly 9 permission bits, divided into three triads:

• Owner (User): The identity owning the file.
• Group: The primary group associated with the file.
• Other: All other system users.

While computationally efficient and requiring only 12 bits of storage per inode (including setuid, setgid, and sticky bits), this compression creates a structural flaw: an owner cannot grant read access to Alice and write access to Bob without creating a dedicated group.

DAC vs. MAC

• Discretionary Access Control (DAC): The resource owner determines access permissions. An unprivileged user can set mode 777 on their home directory or credentials file, exposing it to all users on the host.
• Mandatory Access Control (MAC): Access decisions are governed by a central security policy enforced by the operating system kernel (e.g., SELinux, AppArmor), independent of user intent. Even if a file is mode 777, a MAC policy can block unauthorized process domains from reading it.

Saltzer & Schroeder’s Design Principles (1975)

Four of Jerome Saltzer and Michael Schroeder’s foundational security principles directly govern our analysis of Level 0:

1. Least Privilege: Every program and user should operate using the bare minimum set of privileges necessary. A credentials file should never exceed mode 600.
2. Fail-Safe Defaults: Access decisions must default to denial. Default umask configurations in standard Linux distributions (e.g., 022) create files that are world-readable (644) by default, violating this principle.
3. Complete Mediation: Every access to every object must be checked on every access attempt, preventing cached permission bypasses.
4. Psychological Acceptability: Security mechanisms must not impose excessive cognitive or operational friction. When secure secret management is difficult, administrators store plain passwords in local text files.

Theory: A History of Getting Secrets Wrong

From early plaintext shadow files to modern honeytokens and canary credentials.

Sources

• Morris & Thompson, Password Security: A Case History (Communications of the ACM, 1979)
• MITRE Common Weakness Enumeration: CWE-798: Use of Hard-coded Credentials & CWE-312: Cleartext Storage of Sensitive Information
• Paes de Barros, The Honeytoken Approach (SecurityFocus, 2003)
• CyberArk Threat Research, Unpacking the Uber Breach (2022)

The file credentials discovered in Level 0 represents a recurring design failure spanning five decades of computing.

In early Unix (First through Sixth Editions), /etc/passwd was world-readable (644) because programs like ls needed it to map numeric UIDs to username strings. The file contained the hashed password representations.

In 1979, Robert Morris and Ken Thompson published Password Security: A Case History. They demonstrated that an attacker using a PDP-11/70 could compute approximately 800 password hashes per second. Using a dictionary of 250,000 words, an entire system’s password file could be cracked in five minutes.

Their initial defense was the introduction of a salt — a pseudo-random string appended to the password before hashing, preventing global rainbow table attacks. However, as computing hardware accelerated, leaving hashes accessible to unprivileged users became untenable. In 1988, the Morris Worm used automated dictionary attacks against /etc/passwd to propagate across the internet.

The structural remedy was the Shadow Suite: moving the password hashes out of /etc/passwd into /etc/shadow, a file restricted to mode 640 or 600 owned by root:

$ cat /etc/shadow
cat: /etc/shadow: Permission denied

This permission denial is complete mediation working as intended.

The Modern Threat: CWE-798 and Hard-Coded Credentials

While system password hashes were moved to shadow files, application credentials frequently remain in flat files:

• CWE-798: Use of Hard-coded Credentials.
• CWE-312: Cleartext Storage of Sensitive Information.
• CWE-256: Plaintext Storage of a Password.

In 2022, according to published threat research by CyberArk and public disclosures, an adversary compromised an internal network share at Uber. In a PowerShell script stored in a readable folder, the intruder located hard-coded credentials for an administrative service account. Those static credentials provided access to the enterprise’s privileged access management vault. No vulnerability exploit was required; the adversary simply read a file that was exposed through lax discretionary access controls.

Turning the Tables: The Honeytoken Strategy

In 2003, Brazilian researcher Augusto Paes de Barros introduced the concept of the honeytoken: an attractive piece of data (a credential, API key, or database record) that has no legitimate operational purpose.

Defenders deploy canary credentials into filesystem paths where intruders typically hunt (~/.aws/credentials, archive/credentials, .env). When an adversary discovers and attempts to authenticate using the token, the central authentication system triggers an immediate, high-fidelity security alert. Through honeytokens, the attacker's enumeration loop is transformed into a detection tripwire.

Theory: The Theory Behind the Loop

Synthesizing the practical workflow with fundamental computer science principles.

Every command executed in the first sixty seconds of an engagement directly probes a core operating system structure:

1. Who am I? (whoami) →→ Queries the process subject credentials evaluated by the reference monitor.
2. Where am I? (pwd) →→ Establishes the current namespace node in the rooted directory hierarchy.
3. What is here? (ls -la) →→ Probes the directory table and maps visible and hidden inode entries.
4. What can I do? (stat) →→ Evaluates the compressed access matrix governing subject-object interactions.
5. What is the content? (file / cat -A) →→ Treats files as raw byte streams while guarding against terminal control injection.

The Defensive Mirror

For every offensive discovery action, there is a corresponding defensive control:

• Adversary queries process identity →→ Defender monitors suspicious account enumeration via auditd or EDR.
• Adversary enumerates directory entries →→ Defender enforces strict DAC modes and restricts directory traverse bits.
• Adversary hunts for credentials →→ Defender implements automated secrets management (HashiCorp Vault, AWS Secrets Manager) and deploys honeytoken tripwires.

The Quiet Trap

A quiet trap is an operational failure mode that executes without generating an error message, leading the analyst to incorrect conclusions.

$ cat notes.txt | grep 'rotate'
todo: rotate keys

The command appears to succeed. However, because notes.txt originated from a Windows environment, it contains carriage returns (\r). If we extract the token into an environment variable:

$ TOKEN=$(cat notes.txt | head -n 1 | awk '{print $2}')
$ curl -H "Authorization: Bearer $TOKEN" https://api.internal/auth

The HTTP request fails or corrupts the header because the carriage return is treated as a header termination delimiter.

The Rule of Thumb

Whenever inspecting text files on an unknown system, always verify line endings using file or cat -A. If CRLF terminators are detected, sanitize the file using dos2unix or tr -d '\r' before passing values to downstream tools.

Why These Concepts Matter

• Rooted Simplicity: Understanding that directories are simple tables mapping names to inodes demystifies filesystem navigation.
• Security Boundaries: Recognizing that the kernel respects numeric UIDs rather than shell variables prevents reliance on spoofable user-space data.
• Discretionary Vulnerabilities: Discretionary access control places the burden of protection on individual users, making configuration drift and credential exposure inevitable in large environments.
• Detection Engineering: Understanding that enumeration produces a distinct telemetry trail enables defenders to build high-fidelity behavioral detection rules.

Key Takeaways

1. Orientation First: Never run automated scripts before manually confirming identity (whoami) and location (pwd).
2. Always Use ls -la: Default directory listings conceal dot-files where sensitive configuration and credentials reside.
3. Check Directory Execute Bits: Reading a directory requires r, but accessing files inside it requires x.
4. Identify Before Reading: Use file and cat -A to avoid terminal corruption and detect hidden carriage return traps.
5. Credentials in Files Are Real: MITRE ATT&CK technique T1552.001 remains a primary initial compromise vector in real-world breaches.

The BreachLab Journey Continues

Level 0 establishes the orientation baseline. In subsequent levels of the Ghost track, the challenges become progressively more constrained:

• Level 1 introduces unusual filename syntax (spaces, special characters, leading hyphens) that breaks naive shell command lines.
• Subsequent levels introduce restricted shells, custom binaries, network listener inspection, and cron automation.

The orientation loop mastered here — identifying identity, location, and readable surface — remains identical across every level of the wargame and every enterprise network engagement.

Follow along with the complete playlist

Platform

• BreachLab: https://breachlab.org
• Ghost Track Level 0: https://breachlab.org/tracks/ghost/i/0
• Platform Rules: https://breachlab.org/rules
• Support BreachLab: https://liberapay.com/breachlab/donate

BreachLab is an independent platform dedicated to hands-on cybersecurity education. If content built on its tracks earns revenue, consider supporting the platform via Liberapay to keep it free and accessible for everyone.

GitHub Repository

Follow along with the source code and production pipeline tools in the official repository:

Someone-anon-coder - Overview

Follow the Complete Series

Access every written guide and theoretical deep dive in the official Medium publication series:

List: BreachLab · Ghost I | Curated by Someone | Medium

Explore My Other Learning Series

• Arbiter Guide — Advanced Java Systems Architecture
• Arbiter Guide Theory — Theoretical Foundations of Distributed Java
• Cybersecurity — Offensive Security, Wargames, and Threat Analysis
• Python Guide — Python Systems & Automation
• Python Guide Theory — Python Internals, CPython VM, and Runtime Design
• GoLang Guide — High-Performance Concurrency in Go
• GoLang Guide Theory — Go Runtime, Memory Model, and Scheduler Internals
• Matplotlib — Visual Analytics & Data Presentation
• BreachLab · Ghost I — Level-by-Level Practical & Theoretical Walkthroughs

Sources

1. Ritchie, D. M., & Thompson, K. (1974). The UNIX Time-Sharing System. Communications of the ACM, 17(7), 365–375. https://www.nokia.com/bell-labs/about/dennis-m-ritchie/manintro.html
2. Filesystem Hierarchy Standard Group. (2015). Filesystem Hierarchy Standard (FHS 3.0). Linux Foundation. https://refspecs.linuxfoundation.org/fhs
3. Ylönen, T., & Lonvick, C. (2006). The Secure Shell (SSH) Protocol Architecture (RFC 4251). Internet Engineering Task Force. https://www.rfc-editor.org/rfc/rfc4251
4. Saltzer, J. H., & Schroeder, M. D. (1975). The Protection of Information in Computer Systems. Proceedings of the IEEE, 63(9), 1278–1308. https://www.cs.virginia.edu/~evans/cs551/saltzer/
5. Lampson, B. W. (1974). Protection. ACM Operating Systems Review, 8(1), 18–24. https://bwlampson.site/08-Protection/WebPage.html
6. Morris, R., & Thompson, K. (1979). Password Security: A Case History. Communications of the ACM, 22(11), 594–597. https://www.passwordresearch.com/papers/paper41.html
7. Scarfone, K., Souppaya, M., Cody, A., & Orebaugh, A. (2008). Technical Guide to Information Security Testing and Assessment (NIST SP 800–115). National Institute of Standards and Technology. https://csrc.nist.gov/pubs/sp/800/115/final
8. Hutchins, E. M., Cloppert, M. J., & Amin, R. M. (2011). Intelligence-Driven Computer Network Defense Informed by Analysis of Adversary Campaigns and Intrusion Kill Chains. Lockheed Martin Corporation. https://www.lockheedmartin.com/en-us/capabilities/cyber/cyber-kill-chain.html
9. MITRE Corporation. (2015). Discovery (TA0007) — Enterprise Matrix. MITRE ATT&CK. https://attack.mitre.org/tactics/TA0007/
10. MITRE Corporation. (2006). CWE-798: Use of Hard-coded Credentials. Common Weakness Enumeration. https://cwe.mitre.org/data/definitions/798.html
11. CyberArk Labs. (2022). Unpacking the Uber Breach: Technical Threat Analysis. CyberArk Software. https://www.cyberark.com/resources/threat-research/unpacking-the-uber-breach

This article was prepared with AI assistance. The companion video’s narration is synthesized speech and the presenter is an animated illustration, not a real person. This guide teaches methodology and intentionally does not reveal wargame solutions or passwords.

What’s Next?

In the next installment, we tackle Level 1: Name Game. We confront filenames containing whitespace, special escape sequences, and leading dashes, exploring how the shell’s argument parsing treats options and filenames, and how operators manipulate files that seem impossible to open.

Happy hacking!

Linux Enumeration and Access Control: What Happens in the First Sixty Seconds on a Target was originally published in InfoSec Write-ups on Medium, where people are continuing the conversation by highlighting and responding to this story.

Here’s the source code for today’s lesson:

import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

public class java_14StreamsOptional {
    public static void main(String[] args) {
        List<String> words = List.of("apple", "kiwi", "banana", "fig", "cherry", "date");
        
        String str = words.stream()
            .filter(word -> word.length() > 3)
            .map(String::toUpperCase)
            .collect(Collectors.joining(", "));
        
        int total = words.stream()
            .mapToInt(String::length)
            .reduce(0, Integer::sum);
        
        System.out.println(str);
        System.out.println("Total characters: " + total);
        System.out.println();

        List<Transaction> transactions = List.of(
            new Transaction("Alice", "Groceries", 45.20),
            new Transaction("Bob", "Electronics", 299.99),
            new Transaction("Alice", "Electronics", 150.00),
            new Transaction("Carol", "Groceries", 60.00),
            new Transaction("Alice", "Groceries", 12.50),
            new Transaction("Bob", "Groceries", 22.30)
        );
        
        Map<String, List<Transaction>> customerTransactions = transactions.stream()
            .collect(Collectors.groupingBy(Transaction::customer));

        System.out.println("--- Customer Totals ---");
        System.out.printf("Alice = %.2f%n", totalFor(customerTransactions, "Alice"));
        System.out.printf("Bob = %.2f%n", totalFor(customerTransactions, "Bob"));
        System.out.printf("Dave = %.2f%n", totalFor(customerTransactions, "Dave"));

        Map<String, Double> categoryTotals = transactions.stream()
            .collect(Collectors.groupingBy(
                Transaction::category,
                Collectors.summingDouble(Transaction::amount)
            ));

        System.out.println("\n--- Category Totals ---");
        categoryTotals.forEach((category, totalCount) -> 
            System.out.printf("%s = %.2f%n", category, totalCount)
        );
    }
    
    record Transaction(String customer, String category, double amount) {}

    public static double totalFor(Map<String, List<Transaction>> byCustomer, String customer) {
        return Optional.ofNullable(byCustomer.get(customer))
            .map(list -> list.stream()
            .mapToDouble(Transaction::amount)
            .sum()
        ).orElseGet(() -> {
            System.out.println("No transactions found for " + customer);
            return 0.0;
        });
    }
}

The Day 14 Guide walks through what this program does: a filter → map → collect pipeline, a reduce that sums word lengths, two groupingBy calls over a list of transaction records, and an Optional chain that handles a customer who isn't in the data. This article is about why it behaves the way it does.

As with Day 13, almost none of it started in Java. Mapping a function over a list was in the first LISP paper. Reducing a list with an operator was a single symbol in a 1962 notation. The idea that a sequence can be computed only as it’s consumed was named in 1965 and argued for as a design principle in 1989. Grouping and totaling in one step is SQL’s GROUP BY. And the case for Optional was made, most memorably, by the man who introduced null in the first place. What Java 8 added is a library that brings all of these together, plus a few rules, like "identity" and "use a stream only once," that only make sense once you know where they came from.

If you missed the previous lesson, the Day 13 Guide Theory covers the theory behind lambdas, and the Day 13 Guide covers the code.

map, filter, and reduce: An Old Vocabulary

Operations on whole lists, not on indexes

From LISP’s maplist and APL’s reduction to Bird’s theory of lists

The lesson’s first pipeline never mentions an index:

String str = words.stream()
    .filter(word -> word.length() > 3)
    .map(String::toUpperCase)
    .collect(Collectors.joining(", "));

The idea of applying a function to every element of a list, as one operation, goes back to John McCarthy’s 1960 LISP paper, “Recursive Functions of Symbolic Expressions and Their Computation by Machine, Part I” (Communications of the ACM, 3(4)). Alongside LAMBDA, which Day 13 traced to Church, the paper defined maplist, a function that takes another function and applies it across a list. That is map, about fifty years before java.util.stream.

The second pipeline’s operation has an equally old ancestor:

int total = words.stream()
    .mapToInt(String::length)
    .reduce(0, Integer::sum);

Kenneth Iverson’s A Programming Language (Wiley, 1962), the book that became APL, gave this its own operator: reduction, written /. In APL, +/ applied to a list of numbers adds them all together. That's exactly what reduce(..., Integer::sum) does: it places the + between every pair of elements.

These operations were later given a mathematical theory. Richard Bird’s “An Introduction to the Theory of Lists” (in Logic of Programming and Calculi of Discrete Design, Springer, 1987) treated map, filter, and reduction as algebraic objects with laws, so programs built from them could be reasoned about like equations. Graham Hutton's "A Tutorial on the Universality and Expressiveness of Fold" (Journal of Functional Programming, 9(4), 1999) went further and showed that a single operation, fold, is enough to express a wide range of list functions, including map and filter themselves. Java's reduce is a fold.

What reduce Assumes: Identity and Associativity

The rules in the Javadoc are algebra, not style

Why the starting value must change nothing, and the run that proves it

reduce(0, Integer::sum) takes two arguments, and the Stream.reduce documentation places conditions on both. The first, 0, must be an identity for the accumulator: combining it with any value x must give back x. The second, Integer::sum, must be associative: (a + b) + c must equal a + (b + c).

In algebra, a set with an associative operation and an identity element is called a monoid. Integers under addition with 0 are the textbook example. Bird's 1987 theory relied on exactly these two properties, and they're why reduce asks for them. With associativity, a list can be split into pieces, each piece reduced separately, and the results combined, without changing the answer. That is precisely what a parallel stream does.

The identity rule looks pedantic until it’s broken. Running sequentially, an identity of 10 just looks like "start counting from 10":

seq identity 0  = 28
par identity 0  = 28
seq identity 10 = 38
par identity 10 = 88

With 0, sequential and parallel agree. With 10, they don't. In parallel, the stream split the six words into pieces, and each piece started its own sum from the identity. Here, each of the six elements ended up with its own 10: 28 + 6 × 10 = 88. The exact number depends on how the stream happens to split, which is the problem: a value that isn't really an identity makes the result depend on scheduling. The lesson uses 0, a true identity for +, so its reduce is correct in both modes, even though it only ever runs sequentially.

Laziness: Why Nothing Runs Until a Terminal Operation Asks

A stream is a promise to compute, not a computation

Landin’s streams, lazy evaluators, and Hughes on why it matters

The Guide ran the first pipeline without a terminal operation, with prints inside each lambda, and nothing printed at all. With collect added, the trace looked like this:

filter apple
map apple
filter kiwi
map kiwi
...
filter fig
filter cherry
map cherry

Both behaviors come from a single design decision. The java.util.stream package documentation states it directly: streams are lazy, and computation on the source only happens once the terminal operation starts.

The word stream itself comes from Peter Landin. In “A Correspondence Between ALGOL 60 and Church’s Lambda-Notation” (Communications of the ACM, 8(2), 1965), Landin described streams as lists whose later elements are not computed until they’re needed. In 1976, two papers made that idea a general evaluation strategy: Daniel Friedman and David Wise’s “CONS Should Not Evaluate Its Arguments” (ICALP 1976) and Peter Henderson and James Morris’s “A Lazy Evaluator” (POPL 1976). Both argued that building a data structure should not force its contents to be computed. Abelson and Sussman later made streams a central chapter of Structure and Interpretation of Computer Programs (MIT Press, 1985, §3.5), where they’re built from exactly this kind of delayed list.

The trace shows the payoff. Elements move through the whole pipeline one at a time: apple is filtered and mapped before kiwi is looked at. No intermediate "filtered list" is ever built. Philip Wadler named the compiler version of this optimization deforestation, in "Deforestation: Transforming Programs to Eliminate Trees" (Theoretical Computer Science, 73(2), 1990): when one list operation feeds another, the intermediate structure can be removed entirely. Java's streams get the same effect by design rather than by compiler transformation, because each element is pulled through the chain on demand.

John Hughes’s “Why Functional Programming Matters” (The Computer Journal, 32(2), 1989) explained why laziness is worth having beyond saving work. He called it a kind of glue: it lets you write the producer of data and the consumer of data as separate pieces, and the consumer decides how much is actually computed. In the lesson, filter and map describe what should happen, and collect decides that it happens. fig never reaching map is the smallest possible demonstration.

Why a stream can only be used once

The Guide also showed that a second terminal operation on the same stream throws IllegalStateException. That follows from the same model. A Java stream is not a lazy list in Landin's sense, where computed elements are kept and can be revisited. It's closer to an iterator: a one-way cursor over its source that's consumed as it's pulled. Once the terminal operation has pulled every element through, there's nothing left to pull. The package documentation states the rule plainly: a stream should be operated on only once. The source, words, is untouched, which is why the lesson can call words.stream() again for the next pipeline.

Records: Data as a Named Tuple

A class that is only its data

JEP 395 and the product types of typed functional languages

The transactions in the lesson are a single line:

record Transaction(String customer, String category, double amount) {}

Running javap -p on the compiled class shows what that line produced:

final class java_14StreamsOptional$Transaction extends java.lang.Record {
  private final java.lang.String customer;
  private final java.lang.String category;
  private final double amount;
  java_14StreamsOptional$Transaction(java.lang.String, java.lang.String, double);
  public final java.lang.String toString();
  public final int hashCode();
  public final boolean equals(java.lang.Object);
  public java.lang.String customer();
  public java.lang.String category();
  public double amount();
}

The class is final, every field is private final, and equals, hashCode, and toString are generated from the components, which is everything Day 5 wrote by hand. Records were finalized in Java 16 by JEP 395, after previews in Java 14 and 15. Brian Goetz's design note "Data Classes and Sealed Types for Java" (2019) framed them as nominal tuples: a fixed group of values with names, whose identity is entirely its data.

In type theory, a group of values held together is a product type, and typed functional languages had them as a core feature long before Java. The language Hope, described by Rod Burstall, David MacQueen, and Don Sannella in “HOPE: An Experimental Applicative Language” (LISP Conference, 1980), is one of the early languages where user-defined data types of this kind were central. The accessor names, customer() rather than getCustomer(), reflect that view: a component is part of what the record is, not a property hidden behind a getter. That's also what makes Transaction::customer read naturally as the classifier for groupingBy.

groupingBy: SQL’s GROUP BY in a Collector

Partition first, then aggregate each part

Codd’s relations, SEQUEL’s aggregates, and mutable reduction

The lesson’s second grouping does two things at once:

Map<String, Double> categoryTotals = transactions.stream()
    .collect(Collectors.groupingBy(
        Transaction::category,
        Collectors.summingDouble(Transaction::amount)
    ));

Anyone who has written SQL has seen this shape before:

SELECT category, SUM(amount) FROM transactions GROUP BY category;

Edgar Codd’s “A Relational Model of Data for Large Shared Data Banks” (Communications of the ACM, 13(6), 1970) set out the idea of data as relations, sets of tuples, which a List<Transaction> of records closely resembles. Donald Chamberlin and Raymond Boyce's "SEQUEL: A Structured English Query Language" (ACM SIGFIDET Workshop, 1974), the language that became SQL, brought grouping and aggregate functions to that model: split the rows into groups by a column, then compute one value per group. groupingBy follows the same two steps. The classifier, Transaction::category, is the GROUP BY column. The downstream collector, summingDouble(Transaction::amount), is the SUM(amount).

Under the hood, a Collector is a recipe for what the java.util.stream documentation calls mutable reduction. Where reduce builds a new value at each step, a collector supplies a mutable container and folds each element into it. The Collector interface defines this as four functions: a supplier that creates the container, an accumulator that adds one element to it, a combiner that merges two containers (for parallel streams), and an optional finisher. groupingBy is a collector whose container is a map, and whose accumulator hands each element to the downstream collector for its group. That's why it can total while it groups, in a single pass, with no list of transactions ever built per category.

One rule from the documentation matters for the output: groupingBy makes no guarantees about the type, mutability, or ordering of the map it returns. The Guide checked, and it's a HashMap. In relational terms that's no surprise either: Codd's relations are sets, and a GROUP BY without an ORDER BY has no defined row order.

Optional: The Type That Replaces null

Absence as a value, not a missing pointer

Hoare’s “billion-dollar mistake,” the Maybe type, and Moggi’s monads

totalFor begins with a call that might return null:

return Optional.ofNullable(byCustomer.get(customer))

Tony Hoare introduced the null reference in ALGOL W in 1965, and in a 2009 talk at QCon London titled “Null References: The Billion Dollar Mistake,” he described it as his own error: a value that has a type, but can’t do anything that type promises. Every NullPointerException is that mismatch showing up at runtime.

Typed functional languages took the opposite approach. Instead of letting every reference possibly be null, they used a separate type for “maybe there’s a value”: option in ML, and Maybe in Haskell. A function that might find nothing says so in its return type, and the caller has to handle both cases to get the value out. Optional<T> is Java's version of that idea. The Optional Javadoc describes its intended use narrowly: primarily as a method return type, where "no result" is a real possibility and a null would be likely to cause errors. totalFor's lookup is exactly that case.

The way the lesson uses Optional has its own theory. .map(...) applies a function to the value inside, if there is one, and returns another Optional. In category theory this is a functor, and the pattern of chaining such steps was given its programming-language foundation by Eugenio Moggi's "Notions of Computation and Monads" (Information and Computation, 93(1), 1991), then made practical by Philip Wadler's "Monads for Functional Programming" (in Advanced Functional Programming, LNCS 925, 1995). The point of both papers is the same: an effect like "might be missing" can be carried by the type, so the code in the middle of the chain never has to check for it. That's why totalFor contains no if and no isPresent().

Java’s Optional isn't a perfectly faithful version of the theory. Its map has one special rule: if the function returns null, the result is an empty Optional, not an Optional holding null. I checked:

Optional.of("x").map(s -> null)   // Optional.empty

That’s a practical choice for a language that still has null everywhere, but it means map doesn't always obey the functor law that mapping f and then g equals mapping their composition. In the lesson, the mapped function returns a double, which can never be null, so this edge case never arises.

Two Methods, One Type: Call-by-Value vs. Call-by-Need

The quiet trap, viewed from the theory side

ALGOL 60’s call-by-name, Ingerman’s thunks, and Plotkin’s evaluation strategies

The Guide swapped orElseGet for orElse and got this:

No transactions found for Alice
Alice = 207.70
No transactions found for Bob
Bob = 322.29
No transactions found for Dave
Dave = 0.00

The totals are correct and the log is wrong. The cause is an idea older than Java by four decades: evaluation strategy, the rule for when a function’s arguments are computed.

Java uses call-by-value. Before orElse(fallback(c)) runs, Java evaluates fallback(c), prints its message, and passes in the resulting 0.0. Whether orElse ends up using it is irrelevant. The work is already done.

The ALGOL 60 report (Peter Naur, ed., Communications of the ACM, 3(5), 1960) defined a different option, call-by-name, where an argument is not evaluated at the call. It’s re-evaluated each time the function actually uses it. Implementing that required packaging an unevaluated expression into something that could be run later. P. Z. Ingerman named that package in “Thunks: A Way of Compiling Procedure Statements with Some Comments on Procedure Declarations” (Communications of the ACM, 4(1), 1961): a thunk, a small piece of code that computes an argument on demand. Gordon Plotkin’s “Call-by-Name, Call-by-Value and the λ-Calculus” (Theoretical Computer Science, 1(2), 1975) placed both strategies on formal footing, in the same lambda calculus Day 13 started from. The lazy evaluators of 1976 added call-by-need: evaluate on first use, at most once.

orElseGet is call-by-need, written by hand. Its parameter is a Supplier<T>, a lambda with no arguments that produces a value when its get() method is called. That's exactly a thunk. orElseGet calls it only if the Optional is empty, and at most once. orElse takes a plain T, so the caller's argument has already been evaluated by the time orElse sees it. It's the same distinction as the stream pipeline earlier: a description of work versus the work itself.

The theory gap: the type system can’t see the difference

Here are the two signatures:

T orElse(T other)
T orElseGet(Supplier<? extends T> supplier)

Both return a T. Both produce the same value in every case. In Java's type system, nothing distinguishes "computes the fallback only when needed" from "computes it every time and throws it away," because Java's types describe what a computation returns, not what it does along the way. Printing a line, querying a database, or allocating an object are all invisible to the compiler.

That’s a known limit, and type theory has an answer to it. James Lucassen and David Gifford’s “Polymorphic Effect Systems” (POPL 1988) proposed extending types with effects, so a function’s signature records whether it reads, writes, or performs other side effects. In a language with an effect system, fallback(c) would carry its printing effect in its type, and the cost of evaluating it eagerly would be visible at the call site. Java has no such system. Neither did the Day 13 trap, where a handler that discarded an exception's message had the same type as one that logged it. In both cases, the type checker accepts every version, and the difference has to come from the programmer's understanding of when code runs.

Why These Concepts Matter

• map and reduction predate Java by decades: McCarthy's maplist (1960) and Iverson's / operator (1962), later formalized by Bird (1987) and shown by Hutton (1999) to be expressible as fold
• reduce requires an identity and an associative accumulator, the definition of a monoid, because that's what lets a stream be split and recombined. A false identity gave 38 sequentially and 88 in parallel
• Streams are lazy, following Landin’s streams (1965) and the lazy evaluators of 1976. Nothing runs without a terminal operation, and elements pass through one at a time, which avoids intermediate structures in the spirit of Wadler’s deforestation
• A Java stream is a one-shot cursor, not a reusable lazy list, which is why it can be consumed only once
• Records are nominal tuples, Java’s form of a product type, finalized by JEP 395
• groupingBy with a downstream collector is SQL's GROUP BY with an aggregate, implemented as a mutable reduction, and like an unordered GROUP BY, its result has no guaranteed order
• Optional is Java's version of ML's option and Haskell's Maybe, a response to Hoare's null reference, and its map follows the functor pattern Moggi and Wadler put at the center of functional programming
• orElse is call-by-value; orElseGet takes a thunk and is call-by-need. The type system can't see the difference, because Java's types don't track effects

The Arbiter Journey Continues

One concept at a time, this series is building toward Arbiter, a microservices-based test management platform using Java, Spring Boot, databases, testing frameworks, Docker, and AWS. Today’s theory lines up closely with how it will be built. Summarizing test results into pass, fail, and skipped counts per suite is a GROUP BY with an aggregate, whether it's written as a groupingBy collector in Java or as a query against the database, and knowing they're the same operation makes it easier to decide which side should do the work. Spring Data repositories commonly return Optional from lookups, which is the "return type for a possibly missing result" use the Javadoc describes. Records are a natural fit for request and response objects that are only data. And the evaluation-strategy gap matters directly: a fallback that eagerly runs a query on every lookup has the same type as one that runs it only when needed, so the only protection is understanding when code actually runs.

Watch the complete concept in detail

Follow along with the complete Arbiter playlist

GitHub Repository

You can follow the complete learning journey, source code, explanations, and future lessons here:

GitHub - Someone-anon-coder/Arbiter: Welcome to Arbiter - a documented, hands-on journey from Java fundamentals to a deployed microservices platform.

Follow the Complete Arbiter Series

Every Guide and Guide Theory article in this series, collected in one place:

• List: Arbiter (Java) | Curated by Someone | Medium
• List: Arbiter (Java) Theory | Curated by Someone | Medium

Explore My Other Learning Series

If you’re interested in my other programming and technology learning journeys, you may also enjoy:

• List: Cybersecurity | Curated by Someone | Medium
• List: Python Guide | Curated by Someone | Medium
• List: Python Guide - Theory | Curated by Someone | Medium
• List: GoLang Guide | Curated by Someone | Medium
• List: GoLang Guide Theoretical | Curated by Someone | Medium
• List: My Progress - Matplotlib | Curated by Someone | Medium

What’s Next?

In Day 15, we move into Multithreading: Thread and Runnable. Today's parallel reduce was a first glimpse of the problem: once work is split across threads, the order things happen in is no longer fixed, and rules like associativity stop being optional. The theory goes back to the 1960s again, to the first attempts to reason about programs that run at the same time.

Companion guide article: For the hands-on walkthrough of this lesson, see the Day 14 Guide.

Happy coding! ☕

Understanding Java Streams — The Theory Behind Lazy Evaluation, Folds, and the Optional Type was originally published in CodeX on Medium, where people are continuing the conversation by highlighting and responding to this story.

Day 14 of my Arbiter Learning Journey | Why a stream does nothing until a terminal operation asks, how groupingBy can total while it groups, and why orElse runs its fallback even when nothing is missing

Before we begin, here’s the source code for today’s lesson:

import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

public class java_14StreamsOptional {
    public static void main(String[] args) {
        List<String> words = List.of("apple", "kiwi", "banana", "fig", "cherry", "date");
        
        String str = words.stream()
            .filter(word -> word.length() > 3)
            .map(String::toUpperCase)
            .collect(Collectors.joining(", "));
        
        int total = words.stream()
            .mapToInt(String::length)
            .reduce(0, Integer::sum);
        
        System.out.println(str);
        System.out.println("Total characters: " + total);
        System.out.println();

        List<Transaction> transactions = List.of(
            new Transaction("Alice", "Groceries", 45.20),
            new Transaction("Bob", "Electronics", 299.99),
            new Transaction("Alice", "Electronics", 150.00),
            new Transaction("Carol", "Groceries", 60.00),
            new Transaction("Alice", "Groceries", 12.50),
            new Transaction("Bob", "Groceries", 22.30)
        );
        
        Map<String, List<Transaction>> customerTransactions = transactions.stream()
            .collect(Collectors.groupingBy(Transaction::customer));

        System.out.println("--- Customer Totals ---");
        System.out.printf("Alice = %.2f%n", totalFor(customerTransactions, "Alice"));
        System.out.printf("Bob = %.2f%n", totalFor(customerTransactions, "Bob"));
        System.out.printf("Dave = %.2f%n", totalFor(customerTransactions, "Dave"));

        Map<String, Double> categoryTotals = transactions.stream()
            .collect(Collectors.groupingBy(
                Transaction::category,
                Collectors.summingDouble(Transaction::amount)
            ));

        System.out.println("\n--- Category Totals ---");
        categoryTotals.forEach((category, totalCount) -> 
            System.out.printf("%s = %.2f%n", category, totalCount)
        );
    }
    
    record Transaction(String customer, String category, double amount) {}

    public static double totalFor(Map<String, List<Transaction>> byCustomer, String customer) {
        return Optional.ofNullable(byCustomer.get(customer))
            .map(list -> list.stream()
            .mapToDouble(Transaction::amount)
            .sum()
        ).orElseGet(() -> {
            System.out.println("No transactions found for " + customer);
            return 0.0;
        });
    }
}

Last time, lambdas and method references turned behavior into values you could store, pass, and combine. Today, those values get somewhere to go. The Streams API takes a collection and runs it through a chain of operations, like filter, map, reduce, and collect, where every step is just a lambda or a method reference from Day 13. And Optional gives a "maybe there's a value here" result a proper type, instead of a null waiting to blow up.

The interesting part isn’t how short the code gets. It’s three questions hiding behind it. When does a stream actually do its work? How do you turn a list of records into totals per group in one pass? And what’s the real difference between two methods, orElse and orElseGet, that return exactly the same value?

This is Day 14 of my Arbiter Learning Journey, where I’m learning Java from the ground up before eventually building Arbiter, a complete microservices-based test management platform.

In Day 13, I covered lambdas, functional interfaces, method references, and composing behavior with .and() and .andThen(), along with how a fallback can quietly swallow the reason something failed. If you want the theory behind it (lambda calculus, target typing, and how lambdas compile), the Day 13 Guide Theory article covers it.

Today’s Topics

• Stream pipelines: a source, intermediate operations, and one terminal operation
• Intermediate operations: filter and map, and why they're lazy
• Terminal operations: collect(Collectors.joining(...)) and reduce(...)
• Primitive streams: mapToInt and mapToDouble
• Grouping: Collectors.groupingBy, with and without a downstream collector
• Records: a compact data class, and why Transaction::customer works
• Optional: ofNullable, map, and orElseGet versus orElse

A Stream Is a Plan Until a Terminal Operation Runs It

The lesson opens with a pipeline over six words:

List<String> words = List.of("apple", "kiwi", "banana", "fig", "cherry", "date");

String str = words.stream()
    .filter(word -> word.length() > 3)
    .map(String::toUpperCase)
    .collect(Collectors.joining(", "));

Every stream pipeline has the same three parts:

• A source: words.stream() turns the list into a stream of its elements
• Intermediate operations: filter keeps only words longer than 3 characters, and map turns each one into upper case. Each returns a new stream
• A terminal operation: collect(Collectors.joining(", ")) produces an actual result, here one String with the words joined by commas

The lambdas are exactly the ones from Day 13. word -> word.length() > 3 is a Predicate<String>, and String::toUpperCase is a Function<String, String> written as a method reference.

Output:

APPLE, KIWI, BANANA, CHERRY, DATE

fig is the only word with 3 characters or fewer, so it's the only one missing.

Here’s the part that’s easy to miss. filter and map don't do anything when they're called. They only record a step. The work happens when the terminal operation asks for results. I checked this by adding a print inside each lambda and running the pipeline twice: once ending at .map(...), and once ending at .collect(...):

-- no terminal op
-- with terminal op
filter apple
map apple
filter kiwi
map kiwi
filter banana
map banana
filter fig
filter cherry
map cherry
filter date
map date
APPLE, KIWI, BANANA, CHERRY, DATE

Without a terminal operation, the pipeline compiled, ran, and printed nothing. Not a single lambda was called. With one, two more things become visible:

• Elements go through one at a time, not one stage at a time. apple passes through filter and map before kiwi even reaches filter. The stream isn't building a filtered list and then mapping it.
• fig never reaches map. It fails the filter, so toUpperCase is never called on it.

That’s what “lazy” means here: the stream does exactly the work the terminal operation needs, and no more.

One more rule from the same check: a stream can only be used once. Calling a second terminal operation on the same stream object throws:

java.lang.IllegalStateException: stream has already been operated upon or closed

That’s why the lesson calls words.stream() again for the next pipeline instead of reusing the first one.

reduce: Folding a Stream Down to One Value

The second pipeline adds up the length of every word:

int total = words.stream()
    .mapToInt(String::length)
    .reduce(0, Integer::sum);

mapToInt(String::length) turns the Stream<String> into an IntStream, a stream of plain int values instead of boxed Integer objects. Then reduce(0, Integer::sum) folds them into one number. It starts with 0, the identity, and combines it with each element in turn using Integer::sum:

0 + 5 (apple) = 5
5 + 4 (kiwi) = 9
9 + 6 (banana) = 15
15 + 3 (fig) = 18
18 + 6 (cherry) = 24
24 + 4 (date) = 28

Output:

Total characters: 28

Notice that fig counts here. This is a fresh stream over the original, unfiltered words list. The first pipeline didn't change words at all. Streams never modify their source, and two pipelines over the same list don't share anything.

IntStream also has .sum(), which does the same thing in one call. Writing it as reduce shows what sum actually is: a fold with 0 as the starting value and + as the combining step.

groupingBy: Grouping and Totaling in One Pass

The second half of the lesson works with transactions. Each one is a record:

record Transaction(String customer, String category, double amount) {}

A record is a compact, immutable data class. That one line gives you a constructor, private final fields, and an accessor for each component, named after the component itself: customer(), category(), and amount(). There's no get prefix, which is why the method references below read as Transaction::customer rather than Transaction::getCustomer.

Six transactions go into a list, and the first grouping splits them by customer:

Map<String, List<Transaction>> customerTransactions = transactions.stream()
    .collect(Collectors.groupingBy(Transaction::customer));

groupingBy takes a classifier, a function that says which group each element belongs to. Here, Transaction::customer puts every transaction in a bucket under its customer's name. The result is a Map<String, List<Transaction>>: Alice gets three transactions, Bob gets two, and Carol gets one.

The second grouping does more in the same call:

Map<String, Double> categoryTotals = transactions.stream()
    .collect(Collectors.groupingBy(
        Transaction::category,
        Collectors.summingDouble(Transaction::amount)
    ));

The second argument is a downstream collector. It decides what each bucket holds. Without it, each bucket is a List of the matching elements. With Collectors.summingDouble(Transaction::amount), each bucket is a running total instead, so the result is Map<String, Double> directly. There's no separate step afterward that loops over the lists to add them up.

Tracing the data by hand:

• Groceries: 45.20 + 60.00 + 12.50 + 22.30 = 140.00
• Electronics: 299.99 + 150.00 = 449.99

Output:

--- Category Totals ---
Groceries = 140.00
Electronics = 449.99

One thing worth knowing: groupingBy returns a HashMap (I printed getClass() to check), so it makes no promise about the order the categories come out in. Groceries printing first here isn't something to rely on. If order matters, groupingBy has an overload that takes a map factory, like TreeMap::new.

Optional: Handling a Missing Value Without a null Check

Totals per customer go through a helper method:

public static double totalFor(Map<String, List<Transaction>> byCustomer, String customer) {
    return Optional.ofNullable(byCustomer.get(customer))
        .map(list -> list.stream()
        .mapToDouble(Transaction::amount)
        .sum()
    ).orElseGet(() -> {
        System.out.println("No transactions found for " + customer);
        return 0.0;
    });
}

byCustomer.get(customer) returns the customer's list if they're in the map, and null if they aren't. Dave never appears in the data, so for him it's null. Instead of writing if (list == null), the method wraps the result:

• Optional.ofNullable(...) produces an Optional holding the list if there is one, or an empty Optional if the value was null
• .map(...) runs only if a value is present. It turns the list into its summed amount (with mapToDouble and sum, a stream inside the Optional). If the Optional is empty, map skips the lambda and passes the empty Optional along
• .orElseGet(...) unwraps the result. If there's a value, it returns it. If not, it calls the Supplier lambda, which prints a message and returns 0.0

Output:

--- Customer Totals ---
Alice = 207.70
Bob = 322.29
No transactions found for Dave
Dave = 0.00

The “No transactions found” line appears exactly once, and only for Dave. That’s the proof the Supplier ran only when it was needed. For Alice and Bob, it was never called.

There’s no isPresent() and no get() anywhere in this method. That's the point. Optional is meant to be used through methods like map and orElseGet, not as a null check with extra steps.

Running It

Here’s the complete output of the program, exactly as it ran:

APPLE, KIWI, BANANA, CHERRY, DATE
Total characters: 28

--- Customer Totals ---
Alice = 207.70
Bob = 322.29
No transactions found for Dave
Dave = 0.00
--- Category Totals ---
Groceries = 140.00
Electronics = 449.99

Reading it in groups:

• Line 1: a lazy filter → map → collect pipeline over the words
• Line 2: mapToInt and reduce adding up every word's length, unfiltered
• Lines 4–8: groupingBy by customer, with Optional handling the customer who isn't there
• Lines 10–12: groupingBy by category with a summingDouble downstream collector

The Quiet Trap: orElse Runs Its Fallback Every Time

Optional has two methods that look interchangeable: orElse(value) and orElseGet(supplier). Both return the value if there is one, and the fallback if there isn't. So I swapped the lesson's orElseGet for orElse, with the fallback moved into a method that prints the same message:

static double fallback(String c) {
    System.out.println("No transactions found for " + c);
    return 0.0;
}

double t = Optional.ofNullable(m.get(c))
    .map(l -> l.stream().mapToDouble(Transaction::amount).sum())
    .orElse(fallback(c));

Here’s what printed:

No transactions found for Alice
Alice = 207.70
No transactions found for Bob
Bob = 322.29
No transactions found for Dave
Dave = 0.00

Every total is still correct. Alice still gets 207.70, Bob still gets 322.29. But the program now claims Alice and Bob have no transactions, right before printing their totals.

The reason is ordinary Java, not anything special about Optional. orElse takes a value, and Java evaluates a method's arguments before the method runs. So fallback(c) is called for every customer, every time, whether or not the Optional is empty. Its return value is then thrown away whenever there was a real total. orElseGet takes a Supplier, a lambda, and only calls it when the Optional is actually empty.

That’s what makes it quiet. Nothing fails, and the numbers are right. Here the side effect is a misleading log line. In a real system, it could be a database query, a network call, or a new object created for every lookup and discarded every time. The rule of thumb:

• orElse(...) for a fallback that's a plain value or already exists, like 0.0 or a constant
• orElseGet(...) for a fallback that does work, has side effects, or costs something to build

It’s the same family as Day 13’s swallowed exception: everything behaves correctly on paper, and the cost hides in a place you don’t think to look.

Key Takeaway

Streams describe what should happen to data, and the terminal operation decides when it actually does:

• A pipeline is source → intermediate operations → terminal operation. Without a terminal operation, nothing runs
• Intermediate operations are lazy, and elements pass through the whole pipeline one at a time
• A stream can be used only once, and it never modifies its source
• reduce folds a stream into one value from an identity and a combining function
• groupingBy splits elements into buckets by a classifier, and a downstream collector decides what each bucket holds
• Optional replaces null checks with map and orElseGet, and orElse evaluates its fallback even when the value is present

And like the last few lessons, the dangerous case is the one that runs without complaint. Two methods that return the same answer can do very different amounts of work to get there.

The Arbiter Journey Continues

One concept at a time, this series is building toward Arbiter, a microservices-based test management platform using Java, Spring Boot, databases, testing frameworks, Docker, and AWS. Today’s lesson is close to code Arbiter will actually contain. Turning a list of test results into pass, fail, and skipped counts per suite is a groupingBy with a downstream collector, the same shape as the category totals here. Converting entities into API response objects is a map over a stream. Spring Data JPA repositories commonly return Optional from lookups like findById, so "test case not found" becomes an empty Optional rather than a null. And the orElse trap matters directly: a fallback that quietly runs a query on every lookup is exactly the kind of cost that only shows up once a service is under load.

Watch the complete concept in detail

Follow along with the complete Arbiter playlist

GitHub Repository

You can follow the complete learning journey, source code, explanations, and future lessons here:

GitHub - Someone-anon-coder/Arbiter: Welcome to Arbiter - a documented, hands-on journey from Java fundamentals to a deployed microservices platform.

Follow the Complete Arbiter Series

Every Guide and Guide Theory article in this series, collected in one place:

• List: Arbiter (Java) | Curated by Someone | Medium
• List: Arbiter (Java) Theory | Curated by Someone | Medium

Explore My Other Learning Series

If you’re interested in my other programming and technology learning journeys, you may also enjoy:

• List: Cybersecurity | Curated by Someone | Medium
• List: Python Guide | Curated by Someone | Medium
• List: Python Guide - Theory | Curated by Someone | Medium
• List: GoLang Guide | Curated by Someone | Medium
• List: GoLang Guide Theoretical | Curated by Someone | Medium
• List: My Progress - Matplotlib | Curated by Someone | Medium

What’s Next?

In Day 15, we move into Multithreading: Thread and Runnable. So far, every program in this series has done one thing at a time, in order. Next, Java starts running work side by side, and Runnable, a functional interface with a single run() method, means today's lambdas are already the way to hand that work to a thread.

Happy coding! ☕

Companion theory article: For the theoretical background behind this lesson, see the Day 14 Guide Theory piece, published alongside this one. For the previous lesson’s theory, see Day 13 Guide Theory.

Learning Java from Scratch with Arbiter — Streams API: Lazy Pipelines, reduce, groupingBy, and… was originally published in Dev Genius on Medium, where people are continuing the conversation by highlighting and responding to this story.

An Arbiter Learning Journey check-in | Five questions about format, depth, and code, plus what should come after the series, before it moves into Streams

Thirteen days ago I started learning Java from scratch, in public, with one goal: eventually build Arbiter, a microservices-based test management platform using Java, Spring Boot, databases, testing frameworks, Docker, and AWS.

Every lesson so far has produced two articles. The Guide walks through the code: what it does, what it prints, and where it surprised me. The Guide Theory piece asks why it behaves that way, and traces each idea back to the paper, person, or specification it came from.

This post is different. There’s no new lesson and no source file. Before the series moves into the Streams API, I want to step back, say what I think is working and what isn’t, and ask you a few direct questions. If you’ve read even one article in the series, your answer is useful to me.

What I Think Is Working

Verified output, not predicted output. Every output block in the series comes from actually compiling and running the code. When an article claims something about the compiler, I try to show the compiler saying it: the exact error message, or a few lines of javap output. Writing it this way has caught more of my own wrong assumptions than I'd like to admit.

The Quiet Trap. Each lesson ends with one behavior that runs without complaint and is still wrong: a HashMap tie-break that depended on iteration order, an unchecked exception nothing forced anyone to handle, a fallback that kept the program alive but threw away the reason it failed. These have become my favorite part to write, because they're the bugs that actually reach production.

Theory with names attached. Instead of “historically, exceptions were introduced to…”, the Theory articles try to point at the actual source: Goodenough’s 1975 paper on exception handling, Church’s lambda calculus, Backus’s Turing lecture on composition. I think that makes the ideas easier to remember. You may disagree, which is one of my questions below.

What I’m Changing

Being upfront when the roadmap shifts. If you’ve been following the “What’s Next” sections closely, you’ll have noticed that Sets have been previewed and then postponed more than once. The lesson order follows the source files I’m working through, and that order hasn’t always matched what I previewed. From here on, “What’s Next” is a plan, and when the plan changes I’ll say so at the top of the article rather than in a side note.

Consistency across articles. Link formats and closing sections have drifted a little from day to day. I’m tidying that up so every article in the series looks and navigates the same way.

Five Questions for You

These are the decisions I keep going back and forth on. You don’t need to answer all of them. Reply with the number and a sentence, and that already helps.

1. Two articles per lesson, or one? Is the Guide / Theory split useful, or would you rather read one longer article that covers both? Do you read both, or just one of them?

2. Is the theory depth right? The Theory articles go into history, specifications, and bytecode. Is that the reason you read them, or is it more than you need to understand the lesson?

3. Full code, or key excerpts? Right now the complete source file appears at the top of both articles. Would you prefer just the relevant snippets inline, with the full file linked on GitHub?

4. Length. Is a typical Guide article too long, too short, or about right? If you stopped partway through one, where did you stop?

5. What should come next, or alongside? When Arbiter wraps up, or as something to read between lessons, which of these would you actually read?

• Short standalone “Quiet Trap” posts: one Java gotcha, one fix, five minutes
• Spring Boot internals, explained the same way as this series
• Java for testing: JUnit, test design, and building test automation
• Something else (tell me what)

How to Answer

Leave a comment with the question number and your answer, even if it’s one line. Disagreement is especially welcome. If you think a whole section of the format should go, that’s exactly what I want to hear before I write the next thirteen days the same way.

I’ll read every reply, and I’ll write up what changes as a result in a future post.

The Arbiter Journey Continues

The next lesson, Day 14, moves into the Streams API: map, filter, reduce, and collect, built out of the Predicates, Functions, and method references from Day 13.

If you’re new here, these are the best places to start:

YouTube playlist:

GitHub repository:

GitHub - Someone-anon-coder/Arbiter: Welcome to Arbiter - a documented, hands-on journey from Java fundamentals to a deployed microservices platform.

Arbiter Guide:

List: Arbiter (Java) | Curated by Someone | Medium

Arbiter Guide Theory:

List: Arbiter (Java) Theory | Curated by Someone | Medium

Happy coding! ☕

13 Days of Learning Java in Public: What’s Working, What I’m Changing, and What I’d Like Your Take… was originally published in Dev Genius on Medium, where people are continuing the conversation by highlighting and responding to this story.

Arbiter Guide Theory • Day 13 | Where the arrow came from, why Java gave lambdas interfaces instead of function types, why Function can't throw a checked exception, and what the bytecode reveals about method references

Here’s the source code for today’s lesson:

import java.util.function.Function;
import java.util.function.Predicate;

class Employee {
    private String name;
    private double salary;

    public Employee(String name, double salary) {
        this.name = name;
        this.salary = salary;
    }

    public String getName() { return name; }
    public void setName(String name) { this.name = name; }

    public double getSalary() { return salary; }
    public void setSalary(double salary) { this.salary = salary; }

    public boolean isNameNonBlank() {
        return name != null && !name.trim().isEmpty();
    }

    // Static method used for method reference
    public static boolean isSalaryAboveThreshold(Employee employee) {
        return employee.getSalary() > 60000.0;
    }
}

class FormatException extends Exception {
    public FormatException(String message) {
        super(message);
    }
}

@FunctionalInterface
interface EmployeeFormatter {
    String format(Employee employee) throws FormatException;

    default String formatOrDefault(Employee employee, String fallback) {
        try {
            return format(employee);
        } catch (FormatException e) {
            return fallback;
        }
    }
}

public class java_13Lambdas {
    public static void main(String[] args) {
        String[] usernames = new String[] {
            "aayush99", "bob",
            "x", "claude_code",
            "jd", "ninacodes"
        };

        Predicate<String> correctUsername = str -> str.length() >= 5;
        for (int i = 0; i < usernames.length; i++) {
            if (correctUsername.test(usernames[i])) System.out.println(usernames[i]);
        }
        System.out.println();

        Employee[] employees = new Employee[] {
            new Employee("Alice Kim", 72000.0),
            new Employee("bob jones", 45000.0),
            new Employee("CHARLIE NG", 138000.0),
            new Employee("dana park", 61000.0),
            new Employee("Evan Osei", 29500.0),
            // new Employee(null, 61000.0), // Uncommenting this record will trigger exception
        };

        // Arbitrary-object-instance method reference form (Class::instanceMethod)
        Predicate<Employee> nameCheck = Employee::isNameNonBlank; // Used here
        Predicate<Employee> salaryCheck = Employee::isSalaryAboveThreshold; // Static method reference form
        
        // Composed Predicate for Bonus Eligibility
        Predicate<Employee> bonusEligible = salaryCheck.and(nameCheck);

        EmployeeFormatter reportLineFormatter = emp -> {
            if (emp.getName() == null) throw new FormatException("Employee name cannot be null");

            // Arbitrary-object-instance method reference forms used here
            Function<String, String> normalize = String::trim;
            Function<String, String> uppercase = normalize.andThen(String::toUpperCase);
            
            String formattedName = uppercase.apply(emp.getName());
            return String.format("%s - %.1f", formattedName, emp.getSalary());
        };

        System.out.println("=== Bonus Eligible ===");
        for (int i = 0; i < employees.length; i++) {
            Employee employee = employees[i];
            if (bonusEligible.test(employee)) System.out.println(reportLineFormatter.formatOrDefault(employee, "INVALID_RECORD"));
        }
        System.out.println();

        System.out.println("=== All Records ===");
        for (int i = 0; i < employees.length; i++) {
            System.out.println(reportLineFormatter.formatOrDefault(employees[i], "INVALID_RECORD"));
        }
    }
}

The Day 13 Guide walks through what this program does: a Predicate filtering usernames, a custom EmployeeFormatter that can throw a checked exception, two kinds of method reference, and two compositions. This article is about why it behaves the way it does.

Almost nothing in the lesson is new to computer science. The arrow notation descends from a logician’s notation for anonymous functions in the 1930s. The idea that functions can be passed around like numbers was named in a 1967 lecture. Combining small functions into bigger ones was the centerpiece of a 1978 Turing Award lecture. What is specific to Java is how all of this was bolted onto a language that had spent nearly two decades without it, and the compromises that bolting-on required. Those compromises are exactly where the lesson’s compiler errors come from.

If you missed the previous lesson, the Day 12 Guide Theory covers the theory behind generics, and the Day 12 Guide covers the code.

Where the Arrow Comes From: Church’s Lambda Calculus and McCarthy’s LISP

A function with no name, written as a value

The 1930s notation for anonymous functions, and the 1960 language that made it executable

The lesson’s first lambda is one line:

Predicate<String> correctUsername = str -> str.length() >= 5;

In ordinary mathematics, you name a function before you use it: let f(x) = …. Alonzo Church wanted a way to write the function itself, without naming it. His notation, which first appeared in “A Set of Postulates for the Foundation of Logic” (Annals of Mathematics, 1932) and became the basis of “An Unsolvable Problem of Elementary Number Theory” (American Journal of Mathematics, 58(2), 1936), wrote it as λx.M: “the function that takes x and produces M." That Greek letter is where the name lambda comes from.

Java’s str -> str.length() >= 5 is that same idea with different punctuation. str is the bound variable, -> plays the role of the dot, and str.length() >= 5 is the body. The function has no name. It only exists as a value assigned to correctUsername.

Church’s lambda calculus was a theory of computation on paper. John McCarthy brought it into a running language in “Recursive Functions of Symbolic Expressions and Their Computation by Machine, Part I” (Communications of the ACM, 3(4), 1960), the paper that introduced LISP, and borrowed Church’s LAMBDA notation directly for anonymous functions.

The other half of the idea, that a function can be treated like any other value, got its name from Christopher Strachey’s 1967 lecture notes “Fundamental Concepts in Programming Languages” (later published in Higher-Order and Symbolic Computation, 13, 2000). Strachey called values that can be passed as arguments, returned, and stored in variables first-class citizens. Before Java 8, a Java method was not first-class. You couldn’t store isNameNonBlank in a variable. Today's lesson does exactly that.

Target Typing: Why a Lambda Needs Exactly One Abstract Method

Java chose interfaces over function types

Project Lambda’s central decision, and the rule the compiler enforces

Most languages with lambdas have a dedicated function type, something like (String) -> boolean. Java doesn't. A Java lambda has no type of its own at all. It only gets a type from the context it's written in, which the Java Language Specification calls its target type (JLS §15.27.3).

That wasn’t the only design on the table. In 2006, Gilad Bracha, Neal Gafter, James Gosling, and Peter von der Ahé published the BGGA closures proposal, which added structural function types to Java. The design that actually shipped came from JSR 335, “Lambda Expressions for the Java Programming Language,” led by Brian Goetz and finalized with Java SE 8 in March 2014. Goetz’s design documents, published as the “State of the Lambda” series, explain the choice: Java’s libraries were already full of single-method interfaces like Runnable, Comparator, and ActionListener. Making lambdas target those meant every existing API that took such an interface would accept a lambda immediately, without anything being rewritten.

Those interfaces had an informal name in the community, SAM types (Single Abstract Method). JSR 335 formalized them as functional interfaces (JLS §9.8). The rule follows directly: if the target is an interface, the lambda’s body must be the implementation of one method. With two abstract methods, there’s no way to decide which one the body is for. The Guide showed the compiler saying so:

error: incompatible types: Fmt is not a functional interface
    multiple non-overriding abstract methods found in interface Fmt

And with the annotation from JLS §9.6.4.9, the same check moves to the declaration:

error: Unexpected @FunctionalInterface annotation
  Fmt is not a functional interface
    multiple non-overriding abstract methods found in interface Fmt

@FunctionalInterface changes nothing about what the interface is. Any interface with one abstract method is functional whether or not it's annotated. The annotation only asks the compiler to check the claim where it's made, rather than at some distant line where a lambda first tries to use it.

Default methods, and why they don’t break the rule

EmployeeFormatter has two methods, but only one is abstract:

@FunctionalInterface
interface EmployeeFormatter {
    String format(Employee employee) throws FormatException;

    default String formatOrDefault(Employee employee, String fallback) { ... }
}

Default methods came from the same JSR. Goetz’s 2011 draft “Interface Evolution via Virtual Extension Methods” described the problem they solved: Java 8 needed to add methods like forEach and stream to existing interfaces such as Collection without breaking every class that already implemented them. A default method carries its own body, so implementers don't have to write one. And because it already has a body, it doesn't compete with the lambda for "the one method that needs implementing." That's why JLS §9.8 counts only abstract methods.

Why Function Can't Throw a Checked Exception

The gap between Java’s two type systems

Exception transparency, and the feature that didn’t make it into Java 8

The Guide showed this error:

Function<String, String> f = s -> {
    if (s == null) throw new FormatException("x");
    return s;
};

error: unreported exception FormatException; must be caught or declared to be thrown

The rule behind it is JLS §11.2.3: a lambda body may throw a checked exception only if the function type it targets declares that exception (or a supertype) in its throws clause. Function.apply declares nothing. So a lambda targeting Function can throw only unchecked exceptions.

This is a collision between two separate systems in Java. Generics (Day 12) let an interface abstract over the types of its parameters and result. Checked exceptions (Day 11) are part of a method’s signature too, but there’s no ordinary way for a generic interface to say “and whatever checked exceptions the implementation throws.” The general fix is called exception transparency: letting an interface be parameterized by the exceptions its method can throw. The BGGA proposal included a mechanism for it, and exception transparency was discussed during JSR 335’s development, but it was not part of the design that shipped. The built-in interfaces in java.util.function were defined without throws clauses.

That leaves two options when behavior can fail in a checked way: catch and wrap inside the lambda, or define your own functional interface whose method declares the exception. EmployeeFormatter is the second option. It isn't a style preference. It's the type-system answer to a gap the built-ins deliberately leave open.

Method References and η-Conversion

x -> f(x) is just f

What lambda calculus says, and what the bytecode confirms

The lesson uses two method references that look the same but aren’t:

Predicate<Employee> nameCheck = Employee::isNameNonBlank;
Predicate<Employee> salaryCheck = Employee::isSalaryAboveThreshold;

Lambda calculus has a rule for exactly this situation, called η-conversion (eta): a function that does nothing except pass its argument to another function is equivalent to that other function. λx.(f x) is the same as f. A method reference (JLS §15.13) is Java's built-in η-conversion. e -> Employee.isSalaryAboveThreshold(e) collapses to Employee::isSalaryAboveThreshold.

The instance form adds one twist. isNameNonBlank() takes no arguments, yet Predicate.test passes one in. The resolution is that an instance method has a hidden parameter: the object it's called on. Employee::isNameNonBlank treats the incoming Employee as that receiver, so e -> e.isNameNonBlank() collapses to it.

That difference isn’t just a way of describing things. It shows up in the compiled class file. Running javap -v on the lesson's class lists the method handles each reference points to:

REF_invokeStatic   Employee.isSalaryAboveThreshold:(LEmployee;)Z
REF_invokeVirtual  Employee.isNameNonBlank:()Z
REF_invokeVirtual  java/lang/String.trim:()Ljava/lang/String;
REF_invokeVirtual  java/lang/String.toUpperCase:()Ljava/lang/String;

The static reference compiles to REF_invokeStatic with the Employee as an explicit parameter, (LEmployee;). The arbitrary-instance references compile to REF_invokeVirtual with no parameters at all, (), because the receiver isn't a parameter, it's the object the call is made on. The two "forms" from the Guide are literally two different kinds of JVM call.

Composition: Backus’s Functional Forms

Build programs by combining functions, not by writing steps

The 1978 Turing lecture, and why .and() and .andThen() are combinators

The lesson combines its behaviors in two lines:

Predicate<Employee> bonusEligible = salaryCheck.and(nameCheck);
Function<String, String> uppercase = normalize.andThen(String::toUpperCase);

In mathematics, chaining functions is written with the composition operator: (g ∘ f)(x) = g(f(x)). John Backus, the creator of FORTRAN, made composition the centerpiece of his 1977 Turing Award lecture, published as “Can Programming Be Liberated from the von Neumann Style? A Functional Style and Its Algebra of Programs” (Communications of the ACM, 21(8), 1978). His argument was that programs should be built from a small set of functional forms, operations that take functions and return new functions, instead of from sequences of statements that update variables.

.and() and .andThen() are functional forms in exactly Backus's sense. Each takes a function and returns a new function, without running anything:

• normalize.andThen(String::toUpperCase) is toUpperCase ∘ normalize: trim first, then upper-case. (Java also has compose, which runs in the opposite order: f.compose(g) is f ∘ g, applying g first.)
• salaryCheck.and(nameCheck) is the logical conjunction of two predicates. Its documentation specifies that it short-circuits, just like &&: if salaryCheck returns false, nameCheck is never called.

That second point matters for correctness, not just speed. The order of .and() is the order of evaluation. In this program, the salary check doesn't touch the name and the name check is null-safe, so the order doesn't change the result. But a predicate that assumed a non-null name would have to come after the one that checks for it.

How It Actually Runs: invokedynamic

No anonymous classes, no extra class files

JSR 292, the lambda metafactory, and Landin’s closures

Before Java 8, the only way to pass behavior around was an anonymous inner class, and every one of them compiled to its own .class file on disk. Compiling today's lesson produces exactly four class files: java_13Lambdas.class, Employee.class, EmployeeFormatter.class, and FormatException.class. Six pieces of behavior, zero extra classes.

javap shows where they went. Each lambda and method reference in main is a single invokedynamic instruction:

36: invokedynamic #21,  0   // InvokeDynamic #0:test:()Ljava/util/function/Predicate;
163: invokedynamic #69,  0  // InvokeDynamic #1:test:()Ljava/util/function/Predicate;
170: invokedynamic #70,  0  // InvokeDynamic #2:test:()Ljava/util/function/Predicate;
188: invokedynamic #75,  0  // InvokeDynamic #3:format:()LEmployeeFormatter;

invokedynamic was added in Java 7 by JSR 292, "Supporting Dynamically Typed Languages on the Java Platform," led by John Rose. It was originally meant for languages like JRuby. Java 8 reused it for lambdas: the first time each instruction runs, it calls a bootstrap method, java.lang.invoke.LambdaMetafactory.metafactory, which builds an object implementing the target interface at runtime.

The two arrow-syntax lambdas have bodies that need to live somewhere, so the compiler moved them into private methods of the class:

private static java.lang.String lambda$main$1(Employee) throws FormatException;
private static boolean lambda$main$0(java.lang.String);

Two details here connect back to the theory. First, lambda$main$1 carries throws FormatException in its signature, the checked exception from §11.2.3, preserved all the way into bytecode. Second, method references produce no generated method at all. They point straight at isNameNonBlank, trim, and the rest. That's η-conversion again: there's no wrapper function, because x -> f(x) really is just f.

Both generated methods are also static. Peter Landin's "The Mechanical Evaluation of Expressions" (The Computer Journal, 6(4), 1964) introduced the term closure for a function bundled together with the variables it uses from its surroundings. Neither lambda in this lesson uses any outside variables, so there's nothing to bundle, and the compiler doesn't even need an instance to hold them.

The Theory Gap: A Type System Can Force a Catch, Not a Read

The quiet trap, viewed from the theory side

Goodenough’s exception model, and what formatOrDefault throws away

When the commented-out new Employee(null, 61000.0) line is restored, the program finishes normally:

=== All Records ===
ALICE KIM - 72000.0
BOB JONES - 45000.0
CHARLIE NG - 138000.0
DANA PARK - 61000.0
EVAN OSEI - 29500.0
INVALID_RECORD

Everything the type system asked for was done. The lambda declared FormatException, the default method caught it, and the compiler was satisfied. But the message the exception carried, "Employee name cannot be null", is printed nowhere, because the catch block never reads e.

John Goodenough’s “Exception Handling: Issues and a Proposed Notation” (Communications of the ACM, 18(12), 1975), the paper Java’s exception model traces back to, treated an exception as a way for an operation to tell its caller why it couldn’t complete normally. The type is only half of that message. The other half is the information the exception carries. Joshua Bloch makes the same point as a rule in Effective Java (3rd ed., 2018), Item 77, “Don’t ignore exceptions”: an empty or uninformative handler defeats the purpose of having the exception at all.

That’s the limit of what checked exceptions can do. JLS §11.2.3 can force a lambda’s exception to be caught somewhere. It cannot force the handler to do anything useful with it. A fallback value like "INVALID_RECORD" is a legitimate design choice for a report, but at the type level, a handler that logs the reason and a handler that discards it look identical. The compiler can't tell them apart, so the difference has to come from the programmer.

Why These Concepts Matter

• Lambda calculus (Church, 1932–1936) is where anonymous functions come from, and first-class functions (Strachey, 1967) is the idea that they can be stored and passed like values
• Java lambdas have no type of their own. They take the type of a functional interface, a design JSR 335 chose so that existing single-method APIs would accept lambdas immediately
• Default methods don’t count toward the one-abstract-method rule, because they already have a body
• The built-in interfaces can’t throw checked exceptions because Java 8 shipped without exception transparency. A custom functional interface is the type-system workaround
• A method reference is η-conversion, and the static vs. instance distinction is visible in bytecode as REF_invokeStatic vs. REF_invokeVirtual
• .and() and .andThen() are functional forms in Backus's sense: they combine functions into new functions without running them
• Lambdas compile to invokedynamic and LambdaMetafactory, not anonymous classes
• A checked exception can be forced to be caught, but not read. That gap is where the quiet trap lives

The Arbiter Journey Continues

One concept at a time, this series is building toward Arbiter, a microservices-based test management platform using Java, Spring Boot, databases, testing frameworks, Docker, and AWS. The theory from today shows up throughout it. Spring’s APIs are full of functional interfaces, so configuration and callbacks are often a lambda away. Filtering test runs by several criteria is predicate composition, where the order of .and() decides which check runs first. Steps that call external services and can fail with checked exceptions are exactly where a custom functional interface, rather than a built-in one, earns its place. And the theory gap matters directly: a test platform's whole job is to explain failures, so a handler that discards the reason a step failed would undermine the reason Arbiter exists.

Watch the full explaination into Lambdas

Follow along with the complete Arbiter playlist

GitHub Repository

You can follow the complete learning journey, source code, explanations, and future lessons here:

GitHub - Someone-anon-coder/Arbiter: Welcome to Arbiter - a documented, hands-on journey from Java fundamentals to a deployed microservices platform.

Follow the Complete Arbiter Series

Every Guide and Guide Theory article in this series, collected in one place:

• List: Arbiter (Java) | Curated by Someone | Medium
• List: Arbiter (Java) Theory | Curated by Someone | Medium

Explore My Other Learning Series

If you’re interested in my other programming and technology learning journeys, you may also enjoy:

• List: Cybersecurity | Curated by Someone | Medium
• List: Python Guide | Curated by Someone | Medium
• List: Python Guide - Theory | Curated by Someone | Medium
• List: GoLang Guide | Curated by Someone | Medium
• List: GoLang Guide Theoretical | Curated by Someone | Medium
• List: My Progress - Matplotlib | Curated by Someone | Medium

What’s Next?

In Day 14, we move into the Streams API: map, filter, reduce, and collect. The theory picks up right where today leaves off. map and reduce come straight from the functional-programming tradition Backus argued for, and streams add an idea today's compositions only hinted at: laziness, where nothing runs until a result is actually asked for.

Companion guide article: For the hands-on walkthrough of this lesson, see the Day 13 Guide.

Happy coding! ☕

Understanding Java Lambdas — The Theory Behind Lambda Calculus, Functional Interfaces, and Method… was originally published in CodeX on Medium, where people are continuing the conversation by highlighting and responding to this story.

Day 13 of my Arbiter Learning Journey | Why a lambda needs exactly one abstract method to land on, why the built-in interfaces can’t throw checked exceptions, and how a fallback can quietly swallow the reason something failed

Before we begin, here’s the source code for today’s lesson:

import java.util.function.Function;
import java.util.function.Predicate;

class Employee {
    private String name;
    private double salary;

    public Employee(String name, double salary) {
        this.name = name;
        this.salary = salary;
    }

    public String getName() { return name; }
    public void setName(String name) { this.name = name; }

    public double getSalary() { return salary; }
    public void setSalary(double salary) { this.salary = salary; }

    public boolean isNameNonBlank() {
        return name != null && !name.trim().isEmpty();
    }

    // Static method used for method reference
    public static boolean isSalaryAboveThreshold(Employee employee) {
        return employee.getSalary() > 60000.0;
    }
}

class FormatException extends Exception {
    public FormatException(String message) {
        super(message);
    }
}

@FunctionalInterface
interface EmployeeFormatter {
    String format(Employee employee) throws FormatException;

    default String formatOrDefault(Employee employee, String fallback) {
        try {
            return format(employee);
        } catch (FormatException e) {
            return fallback;
        }
    }
}

public class java_13Lambdas {
    public static void main(String[] args) {
        String[] usernames = new String[] {
            "aayush99", "bob",
            "x", "claude_code",
            "jd", "ninacodes"
        };

        Predicate<String> correctUsername = str -> str.length() >= 5;
        for (int i = 0; i < usernames.length; i++) {
            if (correctUsername.test(usernames[i])) System.out.println(usernames[i]);
        }
        System.out.println();

        Employee[] employees = new Employee[] {
            new Employee("Alice Kim", 72000.0),
            new Employee("bob jones", 45000.0),
            new Employee("CHARLIE NG", 138000.0),
            new Employee("dana park", 61000.0),
            new Employee("Evan Osei", 29500.0),
            // new Employee(null, 61000.0), // Uncommenting this record will trigger exception
        };

        // Arbitrary-object-instance method reference form (Class::instanceMethod)
        Predicate<Employee> nameCheck = Employee::isNameNonBlank; // Used here
        Predicate<Employee> salaryCheck = Employee::isSalaryAboveThreshold; // Static method reference form
        
        // Composed Predicate for Bonus Eligibility
        Predicate<Employee> bonusEligible = salaryCheck.and(nameCheck);

        EmployeeFormatter reportLineFormatter = emp -> {
            if (emp.getName() == null) throw new FormatException("Employee name cannot be null");

            // Arbitrary-object-instance method reference forms used here
            Function<String, String> normalize = String::trim;
            Function<String, String> uppercase = normalize.andThen(String::toUpperCase);
            
            String formattedName = uppercase.apply(emp.getName());
            return String.format("%s - %.1f", formattedName, emp.getSalary());
        };

        System.out.println("=== Bonus Eligible ===");
        for (int i = 0; i < employees.length; i++) {
            Employee employee = employees[i];
            if (bonusEligible.test(employee)) System.out.println(reportLineFormatter.formatOrDefault(employee, "INVALID_RECORD"));
        }
        System.out.println();

        System.out.println("=== All Records ===");
        for (int i = 0; i < employees.length; i++) {
            System.out.println(reportLineFormatter.formatOrDefault(employees[i], "INVALID_RECORD"));
        }
    }
}

Last time, generics let the compiler know exactly what type a class or method was working with. Today, that same compiler learns to treat behavior as a value. Lambda expressions let you write a small piece of logic, like “is this username at least 5 characters long?”, and hand it around like any other object: store it in a variable, pass it to a method, combine it with another one.

The interesting part isn’t the arrow syntax. It’s the three questions hiding behind it. Where does a lambda’s type come from? What happens when the built-in interfaces can’t describe what your lambda needs to do? And how do small pieces of behavior combine without knowing about each other?

This is Day 13 of my Arbiter Learning Journey, where I’m learning Java from the ground up before eventually building Arbiter, a complete microservices-based test management platform.

In Day 12, I covered bounded type parameters, ? extends and ? super wildcards, and why a raw type compiles with only a note and crashes later. If you want the theory behind it (bounded quantification, use-site variance, and erasure), the Day 12 Guide Theory article covers it.

Today’s Topics

• Lambda expressions: str -> str.length() >= 5 as a value you can store and pass
• Functional interfaces: why a lambda needs a target with exactly one abstract method
• Built-in functional interfaces: Predicate and Function from java.util.function
• Custom functional interfaces: @FunctionalInterface, and when the built-ins aren't enough
• Method references: Employee::isSalaryAboveThreshold, Employee::isNameNonBlank, String::trim
• Composition: Predicate.and(...) and Function.andThen(...)

A Lambda Needs Exactly One Abstract Method to Land On

The lesson opens with the smallest possible lambda:

Predicate<String> correctUsername = str -> str.length() >= 5;
for (int i = 0; i < usernames.length; i++) {
    if (correctUsername.test(usernames[i])) System.out.println(usernames[i]);
}

Nothing on the right side of = says what str is. The compiler works it out from the left side. Predicate<String> is an interface with one abstract method, boolean test(String t). So the lambda's single parameter must be a String, and its body must produce a boolean. str.length() >= 5 does. The lambda becomes the implementation of test.

Output:

aayush99
claude_code
ninacodes

bob, x, and jd are all shorter than 5 characters, so test returns false for them.

That “one abstract method” rule isn’t a style guideline. It’s what makes the whole thing work. An interface like that is called a functional interface. If it had two abstract methods, the compiler would have no way to know which one the lambda’s body is supposed to implement. I checked what actually happens with an interface that has two:

interface Fmt { String format(String s); String other(String s); }

Fmt f = s -> s;

error: incompatible types: Fmt is not a functional interface
    multiple non-overriding abstract methods found in interface Fmt

Notice where that error shows up: at the line that writes the lambda, not at the interface. That’s where @FunctionalInterface comes in, and we'll see it next.

A Custom Functional Interface Earns Its Place

Java ships plenty of ready-made functional interfaces. The four most common are Function<T, R> (take a T, return an R), Predicate<T> (take a T, return a boolean), Consumer<T> (take a T, return nothing), and Supplier<T> (take nothing, return a T). So why does the lesson define its own?

@FunctionalInterface
interface EmployeeFormatter {
    String format(Employee employee) throws FormatException;

    default String formatOrDefault(Employee employee, String fallback) {
        try {
            return format(employee);
        } catch (FormatException e) {
            return fallback;
        }
    }
}

The key is throws FormatException. FormatException extends Exception, so it's a checked exception, the kind Day 11 showed the compiler insisting you handle. None of the built-in interfaces declare a throws clause on their abstract method. Function.apply can't throw a checked exception, so a lambda that wants to throw one has nowhere to put it. I tried it:

Function<String, String> f = s -> {
    if (s == null) throw new FormatException("x");
    return s;
};

error: unreported exception FormatException; must be caught or declared to be thrown

So the custom interface isn’t decoration. It’s the only way to say “formatting can fail, and the caller has to deal with that.”

Two more details matter here:

• formatOrDefault is a default method. Default and static methods don't count toward the one-abstract-method limit, so EmployeeFormatter can carry a helper method and still accept a lambda.
• @FunctionalInterface asks the compiler to enforce the rule at the declaration. When I put that annotation on the two-method Fmt from earlier, the error moved from the lambda line to the interface itself:

error: Unexpected @FunctionalInterface annotation
  Fmt is not a functional interface
    multiple non-overriding abstract methods found in interface Fmt

Same idea as yesterday’s bound: tell the compiler your intent once, at the declaration, and it catches mistakes there instead of at some distant call site.

The lambda that implements it is the longest one in the file:

EmployeeFormatter reportLineFormatter = emp -> {
    if (emp.getName() == null) throw new FormatException("Employee name cannot be null");

    Function<String, String> normalize = String::trim;
    Function<String, String> uppercase = normalize.andThen(String::toUpperCase);
    
    String formattedName = uppercase.apply(emp.getName());
    return String.format("%s - %.1f", formattedName, emp.getSalary());
};

A lambda body can be a full block with braces, statements, and a return. Here it can also throw FormatException, because the abstract method it implements declares it.

Method References: Whose Method Is Being Called?

Some lambdas do nothing except call one existing method. For those, Java has a shorter form: the method reference, written with ::.

Predicate<Employee> nameCheck = Employee::isNameNonBlank;
Predicate<Employee> salaryCheck = Employee::isSalaryAboveThreshold;

They look identical, but they’re two different forms. The difference is whose method gets called.

Static method reference. isSalaryAboveThreshold is declared static and takes an Employee as its argument:

public static boolean isSalaryAboveThreshold(Employee employee) {
    return employee.getSalary() > 60000.0;
}

So Employee::isSalaryAboveThreshold is shorthand for e -> Employee.isSalaryAboveThreshold(e). The Employee passed into test is handed straight to the static method as its argument.

Instance method of an arbitrary object. isNameNonBlank is an ordinary instance method that takes no arguments:

public boolean isNameNonBlank() {
    return name != null && !name.trim().isEmpty();
}

There’s no particular Employee object sitting in scope when Employee::isNameNonBlank is written. Instead, the Employee passed into test becomes the object the method is called on: e -> e.isNameNonBlank(). The same form shows up inside the formatter, String::trim means s -> s.trim(), and String::toUpperCase means s -> s.toUpperCase().

The tell is simple: if the name before :: is a type and the method is an instance method, the lambda's parameter becomes the receiver. If it's a static method, the parameter becomes an argument.

There’s also a third form the lesson doesn’t use: a bound instance reference, where a specific object already exists before the ::, like System.out::println (meaning x -> System.out.println(x), always on that one System.out object).

Composition: Small Behaviors, One Decision

Here’s where storing behavior in variables pays off. salaryCheck and nameCheck each answer one question. Combining them takes one line:

Predicate<Employee> bonusEligible = salaryCheck.and(nameCheck);

.and(...) returns a brand-new Predicate that's true only when both are true. Like &&, it short-circuits: for bob jones, the salary check fails, so the name check never runs. Neither original predicate knows the other exists, and both stay usable on their own.

Function has its own version. Inside the formatter:

Function<String, String> normalize = String::trim;
Function<String, String> uppercase = normalize.andThen(String::toUpperCase);

.andThen(...) runs normalize first, then feeds its result into String::toUpperCase. So uppercase.apply("dana park") trims, then upper-cases, producing DANA PARK.

Both compositions are doing real work in the program. bonusEligible decides which records print under "Bonus Eligible":

System.out.println("=== Bonus Eligible ===");
for (int i = 0; i < employees.length; i++) {
    Employee employee = employees[i];
    if (bonusEligible.test(employee)) System.out.println(reportLineFormatter.formatOrDefault(employee, "INVALID_RECORD"));
}

Tracing the data by hand: salary above 60000 and a non-blank name gives Alice Kim (72000), CHARLIE NG (138000), and dana park (61000). Bob (45000) and Evan (29500) fall below the threshold. The program agrees:

=== Bonus Eligible ===
ALICE KIM - 72000.0
CHARLIE NG - 138000.0
DANA PARK - 61000.0

And uppercase is why every name comes out in capitals regardless of how it was entered: "bob jones", "CHARLIE NG", and "Alice Kim" all end up looking the same.

Running It

Here’s the complete output of the program, exactly as it ran:

aayush99
claude_code
ninacodes

=== Bonus Eligible ===
ALICE KIM - 72000.0
CHARLIE NG - 138000.0
DANA PARK - 61000.0

=== All Records ===
ALICE KIM - 72000.0
BOB JONES - 45000.0
CHARLIE NG - 138000.0
DANA PARK - 61000.0
EVAN OSEI - 29500.0

Reading it in groups:

• Lines 1–3: one Predicate<String> lambda filtering usernames by length
• Lines 5–8: a composed Predicate<Employee> (static + arbitrary-instance method references joined with .and()) picking bonus-eligible employees
• Lines 10–15: the custom EmployeeFormatter lambda formatting every record, with names normalized through .andThen()

The Quiet Trap: A Fallback That Throws Away the Reason

The employee array has one line commented out:

// new Employee(null, 61000.0), // Uncommenting this record will trigger exception

When I uncommented it, nothing crashed. The program ran to completion, and the output changed in exactly one place:

=== Bonus Eligible ===
ALICE KIM - 72000.0
CHARLIE NG - 138000.0
DANA PARK - 61000.0

=== All Records ===
ALICE KIM - 72000.0
BOB JONES - 45000.0
CHARLIE NG - 138000.0
DANA PARK - 61000.0
EVAN OSEI - 29500.0
INVALID_RECORD

That’s formatOrDefault doing its job. The lambda threw FormatException, the default method caught it, and returned "INVALID_RECORD". It also proves the error path is real and reachable, not dead code.

But look at what’s missing. The exception carried a message, "Employee name cannot be null", and it appears nowhere. The catch block never reads e. So the report says a record is invalid, but not which one or why. And in the "Bonus Eligible" section, the null-name employee simply isn't there: their 61000 salary passes salaryCheck, nameCheck correctly rejects them, and they vanish without a trace.

Every piece is behaving correctly. That’s what makes it quiet. A fallback is a decision to keep going, and keeping going is often right for a report. But a fallback that discards the exception’s message turns “here’s exactly what went wrong” into “something went wrong somewhere.” It’s the same family as Day 12’s raw type: nothing stops you, and the information you’d need to debug it is the thing that got lost.

Key Takeaway

Lambdas in Java are behavior with a type, and the type comes from the interface they’re assigned to:

• A lambda needs a functional interface, exactly one abstract method, as its target. default and static methods don't count
• @FunctionalInterface moves the "not a functional interface" error from the lambda site to the declaration
• The built-in interfaces can’t throw checked exceptions. When your behavior can fail in a checked way, a custom functional interface is the right tool, not a stylistic choice
• In a method reference, a static method takes the parameter as an argument; an instance method on a type takes the parameter as its receiver
• .and() and .andThen() build one behavior from smaller ones that stay independently reusable

And like the last few lessons, the dangerous case is the one that runs without complaint. A fallback that swallows an exception keeps the program alive but throws away the reason it needed to.

The Arbiter Journey Continues

One concept at a time, this series is building toward Arbiter, a microservices-based test management platform using Java, Spring Boot, databases, testing frameworks, Docker, and AWS. Lambdas will be everywhere in it. Filtering test runs by status, environment, or date is a natural fit for small Predicates combined with .and() and .or(). Turning entities into API response objects is a Function. Spring itself leans on functional interfaces for configuration and callbacks, and the Streams API, which is where this series goes next, is built entirely out of the pieces from today. And the quiet trap matters directly: a test management platform that reports "INVALID" without saying why is exactly the kind of tool nobody trusts.

Watch the concept in detail

Follow along with the complete Arbiter playlist

GitHub Repository

You can follow the complete learning journey, source code, explanations, and future lessons here:

GitHub - Someone-anon-coder/Arbiter: Welcome to Arbiter - a documented, hands-on journey from Java fundamentals to a deployed microservices platform.

Follow the Complete Arbiter Series

Every Guide and Guide Theory article in this series, collected in one place:

• List: Arbiter (Java) | Curated by Someone | Medium
• List: Arbiter (Java) Theory | Curated by Someone | Medium

Explore My Other Learning Series

If you’re interested in my other programming and technology learning journeys, you may also enjoy:

• List: Cybersecurity | Curated by Someone | Medium
• List: Python Guide | Curated by Someone | Medium
• List: Python Guide - Theory | Curated by Someone | Medium
• List: GoLang Guide | Curated by Someone | Medium
• List: GoLang Guide Theoretical | Curated by Someone | Medium
• List: My Progress - Matplotlib | Curated by Someone | Medium

What’s Next?

In Day 14, we move into the Streams API: map, filter, reduce, and collect. Every one of those operations takes a functional interface as its argument, so today's Predicates, Functions, and method references become the building blocks of whole data pipelines, without a single index-based for loop.

Happy coding! ☕

Companion theory article: For the theoretical background behind this lesson, see the Day 13 Guide Theory piece, published alongside this one. For the previous lesson’s theory, see Day 12 Guide Theory.

Learning Java from Scratch with Arbiter — Lambdas: Functional Interfaces, Method References, and… was originally published in Dev Genius on Medium, where people are continuing the conversation by highlighting and responding to this story.