From ce90e124e648cdb444f1a3e5a7f6f094da58c38c Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:24:54 -0500
Subject: [PATCH 001/129] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 61c9acf..f74e08a 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,11 @@
 # csv-diff
 
 [![PyPI](https://img.shields.io/pypi/v/csv-diff.svg)](https://pypi.org/project/csv-diff/)
-[![Changelog](https://img.shields.io/github/v/release/simonw/csv-diff?include_prereleases&label=changelog)](https://github.com/simonw/csv-diff/releases)
+[![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
 [![Tests](https://github.com/simonw/csv-diff/workflows/Test/badge.svg)](https://github.com/simonw/csv-diff/actions?query=workflow%3ATest)
-[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/csv-diff/blob/main/LICENSE)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
 
-Tool for viewing the difference between two CSV, TSV or JSON files. See [Generating a commit log for San Francisco’s official list of trees](https://simonwillison.net/2019/Mar/13/tree-history/) (and the [sf-tree-history repo commit log](https://github.com/simonw/sf-tree-history/commits)) for background information on this project.
+Python CLI tool and library for comparing CSV database dumps and finding differences.
 
 ## Installation
 

From b5d7fca40ad66a2f44404716570932297c2ecc53 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:25:19 -0500
Subject: [PATCH 002/129] Update README.md

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index f74e08a..1f5d0e7 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,6 @@
 
 [![PyPI](https://img.shields.io/pypi/v/csv-diff.svg)](https://pypi.org/project/csv-diff/)
 [![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
-[![Tests](https://github.com/simonw/csv-diff/workflows/Test/badge.svg)](https://github.com/simonw/csv-diff/actions?query=workflow%3ATest)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
 
 Python CLI tool and library for comparing CSV database dumps and finding differences.

From 960c3063604c4cb12332d77cb6077e88b7aa80ef Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:27:03 -0500
Subject: [PATCH 003/129] Update __init__.py

Simplified some code, removed extraneous singular and plural code.
---
 csv_diff/__init__.py | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/csv_diff/__init__.py b/csv_diff/__init__.py
index 59a2eaf..151de43 100644
--- a/csv_diff/__init__.py
+++ b/csv_diff/__init__.py
@@ -108,9 +108,7 @@ def compare(previous, current, show_unchanged=False):
     return result
 
 
-def human_text(result, key=None, singular=None, plural=None, current=None, extras=None):
-    singular = singular or "row"
-    plural = plural or "rows"
+def human_text(result, key=None, current=None, extras=None):
     title = []
     summary = []
     show_headers = sum(1 for key in result if result[key]) > 1
@@ -137,9 +135,7 @@ def human_text(result, key=None, singular=None, plural=None, current=None, extra
             + [""]
         )
     if result["changed"]:
-        fragment = "{} {} changed".format(
-            len(result["changed"]), singular if len(result["changed"]) == 1 else plural
-        )
+        fragment = "{} rows changed".format(len(result["changed"]))
         title.append(fragment)
         if show_headers:
             summary.append(fragment + "\n")
@@ -165,9 +161,7 @@ def human_text(result, key=None, singular=None, plural=None, current=None, extra
                 change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
-        fragment = "{} {} added".format(
-            len(result["added"]), singular if len(result["added"]) == 1 else plural
-        )
+        fragment = "{} rows added".format(len(result["added"]))
         title.append(fragment)
         if show_headers:
             summary.append(fragment + "\n")
@@ -180,9 +174,7 @@ def human_text(result, key=None, singular=None, plural=None, current=None, extra
         summary.append("\n\n".join(rows))
         summary.append("")
     if result["removed"]:
-        fragment = "{} {} removed".format(
-            len(result["removed"]), singular if len(result["removed"]) == 1 else plural
-        )
+        fragment = "{} rows removed".format(len(result["removed"]))
         title.append(fragment)
         if show_headers:
             summary.append(fragment + "\n")

From 264b8454f7930667ad0443ee6223115a19299ad3 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:31:54 -0500
Subject: [PATCH 004/129] Update cli.py

Added time feature,
added ability to specify the input file encoding and made the new default input file encoding utf-8,
removed singular and plural features,
added output to file option (mainly json),
added ability to name output file,
made help text much better, added examples, fixed bad formatting,
added new aliases for help options and for help like -h and -o, added defaults for options,
added more changes that I don't really remember.
---
 csv_diff/cli.py | 109 +++++++++++++++++++++++++++++++++---------------
 1 file changed, 76 insertions(+), 33 deletions(-)

diff --git a/csv_diff/cli.py b/csv_diff/cli.py
index 261ef6d..fc4df6c 100644
--- a/csv_diff/cli.py
+++ b/csv_diff/cli.py
@@ -1,10 +1,13 @@
 import click
 import json as std_json
+import time as time_module
 from . import load_csv, load_json, compare, human_text
 
-
-@click.command()
-@click.version_option()
+@click.command(
+    context_settings={"help_option_names": ["-h", "--help"]},
+    help="Compare the differences between two CSV or JSON files.",
+    epilog="Example: csv-diff old.csv new.csv --key=Id --output=jsonfile --output-file=diff.json",
+)
 @click.argument(
     "previous",
     type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
@@ -14,28 +17,13 @@
     type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
 )
 @click.option(
-    "--key", type=str, default=None, help="Column to use as a unique ID for each row"
+    "-k", "--key", type=str, default=None, help="Column to use as a unique ID for each row (default: first column if not specified) (Example: --key=Id)"
 )
 @click.option(
     "--format",
-    type=click.Choice(["csv", "tsv", "json"]),
-    default=None,
-    help="Explicitly specify input format (csv, tsv, json) instead of auto-detecting",
-)
-@click.option(
-    "--json", type=bool, default=False, help="Output changes as JSON", is_flag=True
-)
-@click.option(
-    "--singular",
-    type=str,
+    "input_format",
     default=None,
-    help="Singular word to use, e.g. 'tree' for '1 tree'",
-)
-@click.option(
-    "--plural",
-    type=str,
-    default=None,
-    help="Plural word to use, e.g. 'trees' for '2 trees'",
+    help="Explicitly specify input format. Available (csv|tsv|json) (default: auto-detect based on file extension)",
 )
 @click.option(
     "--show-unchanged",
@@ -49,34 +37,89 @@
     multiple=True,
     help="key: format string - define extra fields to display",
 )
-def cli(previous, current, key, format, json, singular, plural, show_unchanged, extras):
-    "Diff two CSV or JSON files"
+@click.option(
+    "--encoding",
+    type=str,
+    default="utf-8",
+    help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) (default: utf-8)",
+)
+@click.option(
+    "--time",
+    "show_time",
+    is_flag=True,
+    help="Measure and display elapsed time for the diff operation",
+)
+@click.option(
+    "-o", "--output",
+    default="readable",
+    show_default=True,
+    help="Output format. Available: (readable|json|jsonfile) (default: readable)",
+)
+@click.option(
+    "--output-file",
+    type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
+    default=None,
+    help="File to write JSON output to (only used with --output=jsonfile)",
+)
+@click.version_option()
+def cli(
+    previous, current, key, input_format,
+    show_unchanged, extras, encoding, show_time, output, output_file
+):
+    """Compare the differences between two CSV or JSON files."""
     dialect = {
         "csv": "excel",
         "tsv": "excel-tab",
     }
 
-    if extras and json:
+    if extras and output in ("json", "jsonfile"):
         raise click.UsageError(
             "Extra fields are not supported in JSON output mode",
             ctx=click.get_current_context(),
         )
 
+    if output == "jsonfile" and not output_file:
+        raise click.UsageError(
+            "--output-file must be specified when --output=jsonfile",
+            ctx=click.get_current_context(),
+        )
+
     def load(filename):
-        if format == "json":
-            return load_json(open(filename), key=key)
-        else:
-            return load_csv(
-                open(filename, newline=""), key=key, dialect=dialect.get(format)
-            )
+        try:
+            if input_format == "json":
+                with open(filename, encoding=encoding) as f:
+                    return load_json(f, key=key)
+            else:
+                with open(filename, newline="", encoding=encoding) as f:
+                    return load_csv(f, key=key, dialect=dialect.get(input_format))
+        except Exception as e:
+            raise click.ClickException(f"Failed to load '{filename}': {e}")
 
+    if show_time:
+        start = time_module.time()
+    # Load previous data first to determine key if not set
     previous_data = load(previous)
+    # If key is not set, use the first column from previous_data
+    if key is None:
+        if previous_data and isinstance(previous_data, dict):
+            first_row = next(iter(previous_data.values()))
+            key = list(first_row.keys())[0]
+        else:
+            raise click.ClickException("Could not determine key column from the previous file.")
+    # Now load current data with the determined key
     current_data = load(current)
-
     diff = compare(previous_data, current_data, show_unchanged)
-    if json:
+
+    if output == "json":
         print(std_json.dumps(diff, indent=4))
+    elif output == "jsonfile":
+        with open(output_file, "w", encoding=encoding) as f:
+            f.write(std_json.dumps(diff, indent=4))
     else:
         print(
-            human_text(diff, key, singular, plural, current=current_data, extras=extras)
+            human_text(diff, key, current=current_data, extras=extras)
         )
+
+    if show_time:
+        end = time_module.time()
+        print(f"\nElapsed time: {end - start:.3f} seconds")

From e43a38f032196abbf4f6d3b1116cefa9a9199360 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:33:05 -0500
Subject: [PATCH 005/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 117ed54..7a9a2ec 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "1.2"
+VERSION = "2"
 
 
 def get_long_description():

From 26aecd0dcddd9a51e20700463502353a0c19282a Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:33:36 -0500
Subject: [PATCH 006/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 7a9a2ec..1e30255 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "2"
+VERSION = "2.0"
 
 
 def get_long_description():

From 499ed27df00d66890199d312bb780434ad5b55a8 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:35:10 -0500
Subject: [PATCH 007/129] Update LICENSE

put in the year into the blank space for it
---
 LICENSE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/LICENSE b/LICENSE
index 261eeb9..0bda937 100644
--- a/LICENSE
+++ b/LICENSE
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2025 Thomas Coyle
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

From af89907bb79b09c3f33f650d95e99f6ff196d2fb Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:35:45 -0500
Subject: [PATCH 008/129] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1f5d0e7..220c829 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# csv-diff
+# db-diff
 
 [![PyPI](https://img.shields.io/pypi/v/csv-diff.svg)](https://pypi.org/project/csv-diff/)
 [![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)

From 27662be0acf32775bf0a7c4b110511e1bb2be11a Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 12:36:16 -0500
Subject: [PATCH 009/129] Update README.md

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 220c829..20c1e1c 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,8 @@
 
 Python CLI tool and library for comparing CSV database dumps and finding differences.
 
+# EVERYTHING BELOW IS PROBABLY WRONG, WILL BE UPDATED LATER
+
 ## Installation
 
     pip install csv-diff

From 76c61cd249af109d37a97b3d281536fde281b22d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:13:00 -0500
Subject: [PATCH 010/129] Update publish.yml

updated python version I think
---
 .github/workflows/publish.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index cac9700..0816390 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -30,7 +30,6 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     needs: [test]
-    environment: release
     permissions:
       id-token: write
     steps:
@@ -38,7 +37,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.12"
+        python-version: "3.13"
         cache: pip
         cache-dependency-path: setup.py
     - name: Install dependencies

From ef0cb776ffbda43ec727d43e4a0089c704ba63ec Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:13:21 -0500
Subject: [PATCH 011/129] Update Dockerfile

updated python version I think
---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 14b3de0..6579abe 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
-FROM python:3.12-alpine
-RUN pip install csv-diff
+FROM python:3.13-alpine
+RUN pip install db-diff
 WORKDIR /files
 ENTRYPOINT ["csv-diff"]
 CMD ["--help"]

From cffd08970b96e4c9a4a111af87a6d8cf9249136f Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:14:36 -0500
Subject: [PATCH 012/129] Update setup.py

updated stuff, probably broke something, idk
---
 setup.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/setup.py b/setup.py
index 1e30255..ce31aba 100644
--- a/setup.py
+++ b/setup.py
@@ -14,11 +14,11 @@ def get_long_description():
 
 
 setup(
-    name="csv-diff",
-    description="Python CLI tool and library for diffing CSV and JSON files",
+    name="db-diff",
+    description="Python CLI tool and library for comparing CSV database dumps and finding differences.",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
-    author="Simon Willison",
+    author="Thomas Coyle",
     version=VERSION,
     license="Apache License, Version 2.0",
     packages=find_packages(),
@@ -27,10 +27,10 @@ def get_long_description():
     extras_require={"test": ["pytest"]},
     entry_points="""
         [console_scripts]
-        csv-diff=csv_diff.cli:cli
+        db-diff=db_diff.cli:cli
     """,
-    tests_require=["csv-diff[test]"],
-    url="https://github.com/simonw/csv-diff",
+    tests_require=["db-diff[test]"],
+    url="https://github.com/datsom1/db-diff",
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",

From 25bba00d455c46454b6909bdbd7bbc433bc7930f Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:15:35 -0500
Subject: [PATCH 013/129] Update README.md

---
 README.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 20c1e1c..e50e8f4 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ Python CLI tool and library for comparing CSV database dumps and finding differe
 
 ## Installation
 
-    pip install csv-diff
+    pip install db-diff
 
 ## Usage
 
@@ -28,9 +28,9 @@ Consider two CSV files:
     1,Cleo,5
     3,Bailey,1
 
-`csv-diff` can show a human-readable summary of differences between the files:
+`db-diff` can show a human-readable summary of differences between the files:
 
-    $ csv-diff one.csv two.csv --key=id
+    $ db-diff one.csv two.csv --key=id
     1 row changed, 1 row added, 1 row removed
 
     1 row changed
@@ -58,7 +58,7 @@ You can also feed it JSON files, provided they are a JSON array of objects where
 
 Use `--show-unchanged` to include full details of the unchanged values for rows with at least one change in the diff output:
 
-    % csv-diff one.csv two.csv --key=id --show-unchanged
+    % db-diff one.csv two.csv --key=id --show-unchanged
     1 row changed
 
       id: 1
@@ -71,7 +71,7 @@ Use `--show-unchanged` to include full details of the unchanged values for rows
 
 You can use the `--json` option to get a machine-readable difference:
 
-    $ csv-diff one.csv two.csv --key=id --json
+    $ db-diff one.csv two.csv --key=id --json
     {
         "added": [
             {
@@ -111,7 +111,7 @@ You can specify additional keys to be displayed in the human-readable format usi
 For example, to output a link to `https://news.ycombinator.com/latest?id={id}` for each item with an ID, you could use this:
 
 ```bash
-csv-diff one.csv two.csv --key=id \
+db-diff one.csv two.csv --key=id \
   --extra latest "https://news.ycombinator.com/latest?id={id}"
 ```
 These extras display something like this:
@@ -155,4 +155,4 @@ Suppose current directory contains two csv files : one.csv two.csv
     
 ## Alternatives
 
-- [csvdiff](https://github.com/aswinkarthik/csvdiff) is a "fast diff tool for comparing CSV files" - you may get better results from this than from `csv-diff` against larger files.
+- [csvdiff](https://github.com/aswinkarthik/csvdiff) is a "fast diff tool for comparing CSV files" - you may get better results from this than from `db-diff` against larger files.

From e77f5a765767e6727b3cbf8a67749fd903dd260c Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:17:50 -0500
Subject: [PATCH 014/129] Update .gitignore


From 691a70fe55c71942a362a94f9a77a31b5f792c53 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:37:21 -0500
Subject: [PATCH 015/129] changed csv to db

---
 csv_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csv_diff/cli.py b/csv_diff/cli.py
index fc4df6c..b4c2b0c 100644
--- a/csv_diff/cli.py
+++ b/csv_diff/cli.py
@@ -6,7 +6,7 @@
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
     help="Compare the differences between two CSV or JSON files.",
-    epilog="Example: csv-diff old.csv new.csv --key=Id --output=jsonfile --output-file=diff.json",
+    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --output-file=diff.json",
 )
 @click.argument(
     "previous",

From 83f658208f4ce0248288e017f2b69cca3da97be8 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:38:32 -0500
Subject: [PATCH 016/129] need to update

---
 tests/test_cli.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_cli.py b/tests/test_cli.py
index 12f4e12..39bec4b 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1,3 +1,4 @@
+# this is all wrong
 from click.testing import CliRunner
 from csv_diff import cli, load_csv
 import csv

From b4501666f1abfbcad53c1ebb146eb37075306ba0 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:39:10 -0500
Subject: [PATCH 017/129] not sure what this does

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index df13fc5..122f995 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,4 @@ venv
 *.egg-info
 .DS_Store
 .schema
-.vscode
+.vscode
\ No newline at end of file

From f2685abbfcfd498e0be705fe5604f9a1e36a2ba5 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 13:49:27 -0500
Subject: [PATCH 018/129] renamed csv_diff folder

---
 {csv_diff => db_diff}/__init__.py | 0
 {csv_diff => db_diff}/cli.py      | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename {csv_diff => db_diff}/__init__.py (100%)
 rename {csv_diff => db_diff}/cli.py (100%)

diff --git a/csv_diff/__init__.py b/db_diff/__init__.py
similarity index 100%
rename from csv_diff/__init__.py
rename to db_diff/__init__.py
diff --git a/csv_diff/cli.py b/db_diff/cli.py
similarity index 100%
rename from csv_diff/cli.py
rename to db_diff/cli.py

From 98c4479c08e1f3623334540011769e3751f96129 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:03:13 -0500
Subject: [PATCH 019/129] changed version in setup

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ce31aba..9fc2b49 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "2.0"
+VERSION = "2.1"
 
 
 def get_long_description():

From b2a80f26dd28ccd13f7cb6630b3731bc8808b4cd Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:08:37 -0500
Subject: [PATCH 020/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 9fc2b49..2747349 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "2.1"
+VERSION = "2.3"
 
 
 def get_long_description():

From 9dce823c9430fd55716fa8fc960d9d26581a7a3f Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:32:22 -0500
Subject: [PATCH 021/129] Removed Test stuff

---
 setup.py                 |   3 -
 tests/__init__.py        |   0
 tests/test_cli.py        | 303 ---------------------------------------
 tests/test_csv_diff.py   | 117 ---------------
 tests/test_human_text.py | 224 -----------------------------
 5 files changed, 647 deletions(-)
 delete mode 100644 tests/__init__.py
 delete mode 100644 tests/test_cli.py
 delete mode 100644 tests/test_csv_diff.py
 delete mode 100644 tests/test_human_text.py

diff --git a/setup.py b/setup.py
index 2747349..9bf5902 100644
--- a/setup.py
+++ b/setup.py
@@ -23,13 +23,10 @@ def get_long_description():
     license="Apache License, Version 2.0",
     packages=find_packages(),
     install_requires=["click", "dictdiffer"],
-    setup_requires=["pytest-runner"],
-    extras_require={"test": ["pytest"]},
     entry_points="""
         [console_scripts]
         db-diff=db_diff.cli:cli
     """,
-    tests_require=["db-diff[test]"],
     url="https://github.com/datsom1/db-diff",
     classifiers=[
         "Development Status :: 4 - Beta",
diff --git a/tests/__init__.py b/tests/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/test_cli.py b/tests/test_cli.py
deleted file mode 100644
index 39bec4b..0000000
--- a/tests/test_cli.py
+++ /dev/null
@@ -1,303 +0,0 @@
-# this is all wrong
-from click.testing import CliRunner
-from csv_diff import cli, load_csv
-import csv
-import pytest
-from .test_csv_diff import ONE, ONE_TSV, TWO, TWO_TSV, THREE, FIVE
-import io
-import json
-from textwrap import dedent
-
-
-@pytest.fixture
-def tsv_files(tmpdir):
-    one = tmpdir / "one.tsv"
-    one.write(ONE_TSV)
-    two = tmpdir / "two.tsv"
-    two.write(TWO_TSV)
-    return str(one), str(two)
-
-
-@pytest.fixture
-def json_files(tmpdir):
-    one = tmpdir / "one.json"
-    one.write(
-        json.dumps(
-            [
-                {"id": 1, "name": "Cleo", "nested": {"foo": 3}, "extra": 1},
-                {"id": 2, "name": "Pancakes", "nested": {"foo": 3}},
-            ]
-        )
-    )
-    two = tmpdir / "two.json"
-    two.write(
-        json.dumps(
-            [
-                {"id": 1, "name": "Cleo", "nested": {"foo": 3, "bar": 5}, "extra": 1},
-                {"id": 2, "name": "Pancakes!", "nested": {"foo": 3}, "extra": 1},
-            ]
-        )
-    )
-    return str(one), str(two)
-
-
-def test_human_cli(tmpdir):
-    one = tmpdir / "one.csv"
-    one.write(ONE)
-    two = tmpdir / "two.csv"
-    two.write(TWO)
-    result = CliRunner().invoke(cli.cli, [str(one), str(two), "--key", "id"])
-    assert 0 == result.exit_code
-    assert (
-        dedent(
-            """
-    1 row changed
-
-      id: 1
-        age: "4" => "5"
-    """
-        ).strip()
-        == result.output.strip()
-    )
-
-
-def test_human_cli_alternative_names(tmpdir):
-    one = tmpdir / "one.csv"
-    one.write(ONE)
-    five = tmpdir / "five.csv"
-    five.write(FIVE)
-    result = CliRunner().invoke(
-        cli.cli,
-        [str(one), str(five), "--key", "id", "--singular", "tree", "--plural", "trees"],
-    )
-    assert 0 == result.exit_code, result.output
-    assert (
-        dedent(
-            """
-    1 tree changed, 2 trees added
-
-    1 tree changed
-
-      id: 1
-        age: "4" => "5"
-
-    2 trees added
-
-      id: 3
-      name: Bailey
-      age: 1
-
-      id: 4
-      name: Carl
-      age: 7
-    """
-        ).strip()
-        == result.output.strip()
-    )
-
-
-def test_human_cli_json(tmpdir):
-    one = tmpdir / "one.csv"
-    one.write(ONE)
-    two = tmpdir / "two.csv"
-    two.write(TWO)
-    result = CliRunner().invoke(cli.cli, [str(one), str(two), "--key", "id", "--json"])
-    assert 0 == result.exit_code
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [{"key": "1", "changes": {"age": ["4", "5"]}}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == json.loads(result.output.strip())
-
-
-def test_tsv_files(tsv_files):
-    one, two = tsv_files
-    result = CliRunner().invoke(
-        cli.cli, [one, two, "--key", "id", "--json", "--format", "tsv"]
-    )
-    assert 0 == result.exit_code
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [{"key": "1", "changes": {"age": ["4", "5"]}}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == json.loads(result.output.strip())
-
-
-def test_json_files(json_files):
-    one, two = json_files
-    result = CliRunner().invoke(
-        cli.cli,
-        [one, two, "--key", "id", "--json", "--format", "json"],
-        catch_exceptions=False,
-    )
-    assert 0 == result.exit_code
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [
-            {"key": 1, "changes": {"nested": ['{"foo": 3}', '{"foo": 3, "bar": 5}']}},
-            {
-                "key": 2,
-                "changes": {"name": ["Pancakes", "Pancakes!"], "extra": [None, 1]},
-            },
-        ],
-        "columns_added": [],
-        "columns_removed": [],
-    } == json.loads(result.output.strip())
-
-
-def test_sniff_format(tsv_files):
-    one, two = tsv_files
-    result = CliRunner().invoke(cli.cli, [one, two, "--key", "id", "--json"])
-    assert 0 == result.exit_code
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [{"key": "1", "changes": {"age": ["4", "5"]}}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == json.loads(result.output.strip())
-
-
-def test_format_overrides_sniff(tsv_files):
-    one, two = tsv_files
-    result = CliRunner().invoke(
-        cli.cli, [one, two, "--key", "id", "--json", "--format", "csv"]
-    )
-    assert 1 == result.exit_code
-
-
-def test_column_containing_dot(tmpdir):
-    # https://github.com/simonw/csv-diff/issues/7
-    one = tmpdir / "one.csv"
-    two = tmpdir / "two.csv"
-    one.write(
-        dedent(
-            """
-    id,foo.bar,foo.baz
-    1,Dog,Cat
-    """
-        ).strip()
-    )
-    two.write(
-        dedent(
-            """
-    id,foo.bar,foo.baz
-    1,Dog,Beaver
-    """
-        ).strip()
-    )
-    result = CliRunner().invoke(
-        cli.cli, [str(one), str(two), "--key", "id", "--json"], catch_exceptions=False
-    )
-    assert 0 == result.exit_code
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [{"key": "1", "changes": {"foo.baz": ["Cat", "Beaver"]}}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == json.loads(result.output.strip())
-
-
-def test_semicolon_delimited(tmpdir):
-    # https://github.com/simonw/csv-diff/issues/6
-    one = tmpdir / "one.csv"
-    two = tmpdir / "two.csv"
-    one.write(
-        dedent(
-            """
-    id;name
-    1;Mark
-    """
-        ).strip()
-    )
-    two.write(
-        dedent(
-            """
-    id;name
-    1;Brian
-    """
-        ).strip()
-    )
-    result = CliRunner().invoke(
-        cli.cli, [str(one), str(two), "--key", "id", "--json"], catch_exceptions=False
-    )
-    assert 0 == result.exit_code
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [{"key": "1", "changes": {"name": ["Mark", "Brian"]}}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == json.loads(result.output.strip())
-
-
-def test_diff_with_extras(tmpdir):
-    one = tmpdir / "one.json"
-    two = tmpdir / "two.json"
-    one.write(
-        json.dumps(
-            [
-                {"id": 1, "name": "Cleo", "type": "dog"},
-                {"id": 2, "name": "Suna", "type": "chicken"},
-            ]
-        )
-    )
-    two.write(
-        json.dumps(
-            [
-                {"id": 2, "name": "Suna", "type": "pretty chicken"},
-                {"id": 3, "name": "Artie", "type": "bunny"},
-            ]
-        )
-    )
-    result = CliRunner().invoke(
-        cli.cli,
-        [
-            str(one),
-            str(two),
-            "--key",
-            "id",
-            "--format",
-            "json",
-            "--extra",
-            "search",
-            "https://www.google.com/search?q={name}",
-        ],
-        catch_exceptions=False,
-    )
-    assert result.exit_code == 0
-    expected = dedent(
-        """
-    1 row changed, 1 row added, 1 row removed
-
-    1 row changed
-
-      id: 2
-        type: "chicken" => "pretty chicken"
-      extras:
-        search: https://www.google.com/search?q=Suna
-
-    1 row added
-
-      id: 3
-      name: Artie
-      type: bunny
-      extras:
-        search: https://www.google.com/search?q=Artie
-
-    1 row removed
-
-      id: 1
-      name: Cleo
-      type: dog
-      extras:
-        search: https://www.google.com/search?q=Cleo
-    """
-    ).strip()
-    assert result.output.strip() == expected
diff --git a/tests/test_csv_diff.py b/tests/test_csv_diff.py
deleted file mode 100644
index 0e3670f..0000000
--- a/tests/test_csv_diff.py
+++ /dev/null
@@ -1,117 +0,0 @@
-from csv_diff import load_csv, compare
-import io
-
-ONE = """id,name,age
-1,Cleo,4
-2,Pancakes,2"""
-
-ONE_TSV = """id\tname\tage
-1\tCleo\t4
-2\tPancakes\t2"""
-
-TWO = """id,name,age
-1,Cleo,5
-2,Pancakes,2"""
-
-TWO_TSV = """id\tname\tage
-1\tCleo\t5
-2\tPancakes\t2"""
-
-THREE = """id,name,age
-1,Cleo,5"""
-
-FOUR = """id,name,age
-1,Cleo,5
-2,Pancakes,2,
-3,Bailey,1"""
-
-FIVE = """id,name,age
-1,Cleo,5
-2,Pancakes,2,
-3,Bailey,1
-4,Carl,7"""
-
-SIX = """id,name,age
-1,Cleo,5
-3,Bailey,1"""
-
-SEVEN = """id,name,weight
-1,Cleo,48
-3,Bailey,20"""
-
-EIGHT = """id,name,age,length
-3,Bailee,1,100
-4,Bob,7,422"""
-
-NINE = """id,name,age
-1,Cleo,5
-2,Pancakes,4"""
-
-TEN = """id,name,age
-1,Cleo,5
-2,Pancakes,3"""
-
-
-def test_row_changed():
-    diff = compare(
-        load_csv(io.StringIO(ONE), key="id"), load_csv(io.StringIO(TWO), key="id")
-    )
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [{"key": "1", "changes": {"age": ["4", "5"]}}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == diff
-
-
-def test_row_added():
-    diff = compare(
-        load_csv(io.StringIO(THREE), key="id"), load_csv(io.StringIO(TWO), key="id")
-    )
-    assert {
-        "changed": [],
-        "removed": [],
-        "added": [{"age": "2", "id": "2", "name": "Pancakes"}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == diff
-
-
-def test_row_removed():
-    diff = compare(
-        load_csv(io.StringIO(TWO), key="id"), load_csv(io.StringIO(THREE), key="id")
-    )
-    assert {
-        "changed": [],
-        "removed": [{"age": "2", "id": "2", "name": "Pancakes"}],
-        "added": [],
-        "columns_added": [],
-        "columns_removed": [],
-    } == diff
-
-
-def test_columns_changed():
-    diff = compare(
-        load_csv(io.StringIO(SIX), key="id"), load_csv(io.StringIO(SEVEN), key="id")
-    )
-    assert {
-        "changed": [],
-        "removed": [],
-        "added": [],
-        "columns_added": ["weight"],
-        "columns_removed": ["age"],
-    } == diff
-
-
-def test_tsv():
-    diff = compare(
-        load_csv(io.StringIO(ONE), key="id"), load_csv(io.StringIO(TWO_TSV), key="id")
-    )
-    assert {
-        "added": [],
-        "removed": [],
-        "changed": [{"key": "1", "changes": {"age": ["4", "5"]}}],
-        "columns_added": [],
-        "columns_removed": [],
-    } == diff
diff --git a/tests/test_human_text.py b/tests/test_human_text.py
deleted file mode 100644
index f1bd381..0000000
--- a/tests/test_human_text.py
+++ /dev/null
@@ -1,224 +0,0 @@
-from csv_diff import load_csv, compare, human_text
-from .test_csv_diff import ONE, TWO, THREE, FOUR, FIVE, SIX, SEVEN, EIGHT, NINE, TEN
-from textwrap import dedent
-import io
-
-
-def test_row_changed():
-    diff = compare(
-        load_csv(io.StringIO(ONE), key="id"), load_csv(io.StringIO(TWO), key="id")
-    )
-    assert (
-        dedent(
-            """
-    1 row changed
-
-      id: 1
-        age: "4" => "5"
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_row_changed_show_unchanged():
-    diff = compare(
-        load_csv(io.StringIO(ONE), key="id"),
-        load_csv(io.StringIO(TWO), key="id"),
-        show_unchanged=True,
-    )
-    assert (
-        dedent(
-            """
-    1 row changed
-
-      id: 1
-        age: "4" => "5"
-
-        Unchanged:
-          name: "Cleo"
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_row_added():
-    diff = compare(
-        load_csv(io.StringIO(THREE), key="id"), load_csv(io.StringIO(TWO), key="id")
-    )
-    assert (
-        dedent(
-            """
-    1 row added
-
-      id: 2
-      name: Pancakes
-      age: 2
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_rows_added():
-    diff = compare(
-        load_csv(io.StringIO(THREE), key="id"), load_csv(io.StringIO(FIVE), key="id")
-    )
-    assert (
-        dedent(
-            """
-    3 rows added
-
-      id: 2
-      name: Pancakes
-      age: 2
-
-      id: 3
-      name: Bailey
-      age: 1
-
-      id: 4
-      name: Carl
-      age: 7
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_row_removed():
-    diff = compare(
-        load_csv(io.StringIO(TWO), key="id"), load_csv(io.StringIO(THREE), key="id")
-    )
-    assert (
-        dedent(
-            """
-    1 row removed
-
-      id: 2
-      name: Pancakes
-      age: 2
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_row_changed_and_row_added_and_row_deleted():
-    "Should have headers for each section here"
-    diff = compare(
-        load_csv(io.StringIO(ONE), key="id"), load_csv(io.StringIO(SIX), key="id")
-    )
-    assert (
-        dedent(
-            """
-    1 row changed, 1 row added, 1 row removed
-
-    1 row changed
-
-      id: 1
-        age: "4" => "5"
-
-    1 row added
-
-      id: 3
-      name: Bailey
-      age: 1
-
-    1 row removed
-
-      id: 2
-      name: Pancakes
-      age: 2
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_columns_changed():
-    diff = compare(
-        load_csv(io.StringIO(SIX), key="id"), load_csv(io.StringIO(SEVEN), key="id")
-    )
-    assert (
-        dedent(
-            """
-    1 column added, 1 column removed
-
-    1 column added
-
-      weight
-
-    1 column removed
-
-      age
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_columns_and_rows_changed():
-    diff = compare(
-        load_csv(io.StringIO(SEVEN), key="id"), load_csv(io.StringIO(EIGHT), key="id")
-    )
-    assert (
-        dedent(
-            """
-    2 columns added, 1 column removed, 1 row changed, 1 row added, 1 row removed
-
-    2 columns added
-
-      age
-      length
-
-    1 column removed
-
-      weight
-
-    1 row changed
-
-      id: 3
-        name: "Bailey" => "Bailee"
-
-    1 row added
-
-      id: 4
-      name: Bob
-      age: 7
-      length: 422
-
-    1 row removed
-
-      id: 1
-      name: Cleo
-      weight: 48
-    """
-        ).strip()
-        == human_text(diff, "id")
-    )
-
-
-def test_no_key():
-    diff = compare(load_csv(io.StringIO(NINE)), load_csv(io.StringIO(TEN)))
-    assert (
-        dedent(
-            """
-        1 row added, 1 row removed
-
-        1 row added
-
-          id: 2
-          name: Pancakes
-          age: 3
-
-        1 row removed
-
-          id: 2
-          name: Pancakes
-          age: 4
-        """
-        ).strip()
-        == human_text(diff)
-    )

From 90897a0d58870c0d914eb1a248d46d090e9ff955 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:34:24 -0500
Subject: [PATCH 022/129] better gitignore

---
 .gitignore | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index 122f995..f107b77 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,19 @@
-.venv
+# Virtual environments
+.venv/
+venv/
+
+# Bytecode and compiled files
 __pycache__/
 *.py[cod]
 *$py.class
-venv
-.eggs
-.pytest_cache
-*.egg-info
+
+# Packaging
+.eggs/
+*.egg-info/
+
+# Testing
+.pytest_cache/
+
+# OS and IDE
 .DS_Store
-.schema
-.vscode
\ No newline at end of file
+.vscode/

From 38aa56d30c94e220124f8394824529aab2319d7b Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:37:30 -0500
Subject: [PATCH 023/129] docker

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 6579abe..b9956cf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 FROM python:3.13-alpine
 RUN pip install db-diff
 WORKDIR /files
-ENTRYPOINT ["csv-diff"]
+ENTRYPOINT ["db-diff"]
 CMD ["--help"]

From 874dda8393fc57e7dc9860ef5c1303d4c7bb8bc8 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:37:52 -0500
Subject: [PATCH 024/129] removed publish and test .github .yml files

---
 .github/workflows/publish.yml | 50 -----------------------------------
 .github/workflows/test.yml    | 27 -------------------
 2 files changed, 77 deletions(-)
 delete mode 100644 .github/workflows/publish.yml
 delete mode 100644 .github/workflows/test.yml

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
deleted file mode 100644
index 0816390..0000000
--- a/.github/workflows/publish.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-name: Publish Python Package
-
-on:
-  release:
-    types: [created]
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: pip
-        cache-dependency-path: setup.py
-    - name: Install dependencies
-      run: |
-        pip install '.[test]'
-    - name: Run tests
-      run: |
-        pytest
-  deploy:
-    runs-on: ubuntu-latest
-    needs: [test]
-    permissions:
-      id-token: write
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.13"
-        cache: pip
-        cache-dependency-path: setup.py
-    - name: Install dependencies
-      run: |
-        pip install setuptools wheel build
-    - name: Build
-      run: |
-        python -m build
-    - name: Publish
-      uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
deleted file mode 100644
index f7d84e1..0000000
--- a/.github/workflows/test.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-name: Test
-
-on: [push, pull_request]
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: pip
-        cache-dependency-path: setup.py
-    - name: Install dependencies
-      run: |
-        pip install '.[test]'
-    - name: Run tests
-      run: |
-        pytest

From cc064743e185cfe6cbe50cd9227a85cd8e4fe8f4 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:40:21 -0500
Subject: [PATCH 025/129] docker local

---
 Dockerfile | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index b9956cf..8749e91 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,12 @@
 FROM python:3.13-alpine
-RUN pip install db-diff
+
+# Copy your local code into the container
+COPY . /app
+WORKDIR /app
+
+# Install your package locally
+RUN pip install .
+
 WORKDIR /files
 ENTRYPOINT ["db-diff"]
 CMD ["--help"]

From cf0f9fc98f426d3aee42b8a3eb828f51753b931c Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:43:08 -0500
Subject: [PATCH 026/129] Update README.md

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index e50e8f4..d895c44 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,5 @@
 # db-diff
 
-[![PyPI](https://img.shields.io/pypi/v/csv-diff.svg)](https://pypi.org/project/csv-diff/)
 [![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
 

From cc0065d6f47f46ac15d39e0ff7ddcf2d6d255d67 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:43:22 -0500
Subject: [PATCH 027/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 9bf5902..4b4634e 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "2.3"
+VERSION = "2.4"
 
 
 def get_long_description():

From 8d82274489eac3fefb39df42a9f643303f83c919 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:53:20 -0500
Subject: [PATCH 028/129] Update README.md

---
 README.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/README.md b/README.md
index d895c44..afb379e 100644
--- a/README.md
+++ b/README.md
@@ -5,11 +5,9 @@
 
 Python CLI tool and library for comparing CSV database dumps and finding differences.
 
-# EVERYTHING BELOW IS PROBABLY WRONG, WILL BE UPDATED LATER
-
 ## Installation
 
-    pip install db-diff
+    pip install git+https://github.com/datsom1/db-diff.git
 
 ## Usage
 

From 53cabbbabc11aa7b003222e82333e6f92a995810 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 14:59:31 -0500
Subject: [PATCH 029/129] Changed --output-file to --filename

---
 db_diff/cli.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index b4c2b0c..ff9e797 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -6,7 +6,7 @@
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
     help="Compare the differences between two CSV or JSON files.",
-    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --output-file=diff.json",
+    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --filename=diff.json",
 )
 @click.argument(
     "previous",
@@ -56,7 +56,7 @@
     help="Output format. Available: (readable|json|jsonfile) (default: readable)",
 )
 @click.option(
-    "--output-file",
+    "--filename",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
     help="File to write JSON output to (only used with --output=jsonfile)",
@@ -80,7 +80,7 @@ def cli(
 
     if output == "jsonfile" and not output_file:
         raise click.UsageError(
-            "--output-file must be specified when --output=jsonfile",
+            "--filename must be specified when --output=jsonfile (example: --filename=diff.json)",
             ctx=click.get_current_context(),
         )
 

From f6b2a97d13224d77f1035d823fbb0bba69da7367 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:10:51 -0500
Subject: [PATCH 030/129] Output is now much more compact allowing for easier
 comparison of tables with lots of columns. Ex: even just 1 change in a table
 with 200 columns would output more than 400 lines before

---
 db_diff/__init__.py | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 151de43..fbdeae0 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -141,24 +141,22 @@ def human_text(result, key=None, current=None, extras=None):
             summary.append(fragment + "\n")
         change_blocks = []
         for details in result["changed"]:
-            block = []
-            block.append("  {}: {}".format(key, details["key"]))
-            for field, (prev_value, current_value) in details["changes"].items():
-                block.append(
-                    '    {}: "{}" => "{}"'.format(field, prev_value, current_value)
-                )
+            # Compact: key: field1: "old" => "new", field2: "old" => "new", ...
+            changes_str = ", ".join(
+                f'{field}: "{prev}" => "{curr}"'
+                for field, (prev, curr) in details["changes"].items()
+            )
+            block = [f"  {key}: {details['key']} | {changes_str}"]
             if extras:
                 current_item = current[details["key"]]
                 block.append(human_extras(current_item, extras))
-            block.append("")
-            change_blocks.append("\n".join(block))
             if details.get("unchanged"):
-                block = []
-                block.append("    Unchanged:")
-                for field, value in details["unchanged"].items():
-                    block.append('      {}: "{}"'.format(field, value))
-                block.append("")
-                change_blocks.append("\n".join(block))
+                unchanged_str = ", ".join(
+                    f'{field}: "{value}"'
+                    for field, value in details["unchanged"].items()
+                )
+                block.append(f"    Unchanged: {unchanged_str}")
+            change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
         fragment = "{} rows added".format(len(result["added"]))
@@ -171,7 +169,7 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n\n".join(rows))
+        summary.append("\n".join(rows))
         summary.append("")
     if result["removed"]:
         fragment = "{} rows removed".format(len(result["removed"]))
@@ -184,16 +182,14 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n\n".join(rows))
+        summary.append("\n".join(rows))
         summary.append("")
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
 
 def human_row(row, prefix=""):
-    bits = []
-    for key, value in row.items():
-        bits.append("{}{}: {}".format(prefix, key, value))
-    return "\n".join(bits)
+    # Compact: key1=value1, key2=value2, ...
+    return prefix + ", ".join(f"{k}={v}" for k, v in row.items())
 
 
 def human_extras(row, extras):

From 2ed1c97a6be4ceaa82ae5bacc2249ebc058943b1 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:13:45 -0500
Subject: [PATCH 031/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4b4634e..397bf66 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "2.4"
+VERSION = "2.6"
 
 
 def get_long_description():

From f48e565434fc92fa2faf92d581d0b1ce9bd2a466 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:29:57 -0500
Subject: [PATCH 032/129] fixed error with outputfile option

---
 db_diff/cli.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index ff9e797..2357c72 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -6,7 +6,7 @@
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
     help="Compare the differences between two CSV or JSON files.",
-    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --filename=diff.json",
+    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json",
 )
 @click.argument(
     "previous",
@@ -56,7 +56,8 @@
     help="Output format. Available: (readable|json|jsonfile) (default: readable)",
 )
 @click.option(
-    "--filename",
+    "--outputfile",
+    "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
     help="File to write JSON output to (only used with --output=jsonfile)",
@@ -80,7 +81,7 @@ def cli(
 
     if output == "jsonfile" and not output_file:
         raise click.UsageError(
-            "--filename must be specified when --output=jsonfile (example: --filename=diff.json)",
+            "--outputfile must be specified when --output=jsonfile",
             ctx=click.get_current_context(),
         )
 

From 275ed9b998d55d5f19f0951c688f5719cb424ea5 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:30:19 -0500
Subject: [PATCH 033/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 397bf66..786c861 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "2.6"
+VERSION = "2.7"
 
 
 def get_long_description():

From 691902826f67b039e0a44c03f18a0babac9482df Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:33:23 -0500
Subject: [PATCH 034/129] updated help text

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 2357c72..95d4564 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -5,7 +5,7 @@
 
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
-    help="Compare the differences between two CSV or JSON files.",
+    help="Compare the differences between two CSV or JSON files to find differences.",
     epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json",
 )
 @click.argument(

From 6cc1dbefd3e909e5e611c4c718a82803e56b0a0e Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:36:06 -0500
Subject: [PATCH 035/129] test

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 95d4564..6c5c56f 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -5,7 +5,7 @@
 
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
-    help="Compare the differences between two CSV or JSON files to find differences.",
+    help="Compare the differences between two CSV or JSON files to find differences. test",
     epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json",
 )
 @click.argument(

From 4ca51d4baa4085f724d6ab0d9f09de66fe043e7d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:39:51 -0500
Subject: [PATCH 036/129] removed test

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 6c5c56f..95d4564 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -5,7 +5,7 @@
 
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
-    help="Compare the differences between two CSV or JSON files to find differences. test",
+    help="Compare the differences between two CSV or JSON files to find differences.",
     epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json",
 )
 @click.argument(

From 1fa088fee985d0343692e574755d157b5619de83 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:42:26 -0500
Subject: [PATCH 037/129] fix?

---
 db_diff/__init__.py | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index fbdeae0..151de43 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -141,22 +141,24 @@ def human_text(result, key=None, current=None, extras=None):
             summary.append(fragment + "\n")
         change_blocks = []
         for details in result["changed"]:
-            # Compact: key: field1: "old" => "new", field2: "old" => "new", ...
-            changes_str = ", ".join(
-                f'{field}: "{prev}" => "{curr}"'
-                for field, (prev, curr) in details["changes"].items()
-            )
-            block = [f"  {key}: {details['key']} | {changes_str}"]
+            block = []
+            block.append("  {}: {}".format(key, details["key"]))
+            for field, (prev_value, current_value) in details["changes"].items():
+                block.append(
+                    '    {}: "{}" => "{}"'.format(field, prev_value, current_value)
+                )
             if extras:
                 current_item = current[details["key"]]
                 block.append(human_extras(current_item, extras))
-            if details.get("unchanged"):
-                unchanged_str = ", ".join(
-                    f'{field}: "{value}"'
-                    for field, value in details["unchanged"].items()
-                )
-                block.append(f"    Unchanged: {unchanged_str}")
+            block.append("")
             change_blocks.append("\n".join(block))
+            if details.get("unchanged"):
+                block = []
+                block.append("    Unchanged:")
+                for field, value in details["unchanged"].items():
+                    block.append('      {}: "{}"'.format(field, value))
+                block.append("")
+                change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
         fragment = "{} rows added".format(len(result["added"]))
@@ -169,7 +171,7 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n".join(rows))
+        summary.append("\n\n".join(rows))
         summary.append("")
     if result["removed"]:
         fragment = "{} rows removed".format(len(result["removed"]))
@@ -182,14 +184,16 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n".join(rows))
+        summary.append("\n\n".join(rows))
         summary.append("")
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
 
 def human_row(row, prefix=""):
-    # Compact: key1=value1, key2=value2, ...
-    return prefix + ", ".join(f"{k}={v}" for k, v in row.items())
+    bits = []
+    for key, value in row.items():
+        bits.append("{}{}: {}".format(prefix, key, value))
+    return "\n".join(bits)
 
 
 def human_extras(row, extras):

From ae2d5543621da39924249036374d746b8d7baf8f Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:43:34 -0500
Subject: [PATCH 038/129] fix for fix

---
 db_diff/__init__.py | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 151de43..fbdeae0 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -141,24 +141,22 @@ def human_text(result, key=None, current=None, extras=None):
             summary.append(fragment + "\n")
         change_blocks = []
         for details in result["changed"]:
-            block = []
-            block.append("  {}: {}".format(key, details["key"]))
-            for field, (prev_value, current_value) in details["changes"].items():
-                block.append(
-                    '    {}: "{}" => "{}"'.format(field, prev_value, current_value)
-                )
+            # Compact: key: field1: "old" => "new", field2: "old" => "new", ...
+            changes_str = ", ".join(
+                f'{field}: "{prev}" => "{curr}"'
+                for field, (prev, curr) in details["changes"].items()
+            )
+            block = [f"  {key}: {details['key']} | {changes_str}"]
             if extras:
                 current_item = current[details["key"]]
                 block.append(human_extras(current_item, extras))
-            block.append("")
-            change_blocks.append("\n".join(block))
             if details.get("unchanged"):
-                block = []
-                block.append("    Unchanged:")
-                for field, value in details["unchanged"].items():
-                    block.append('      {}: "{}"'.format(field, value))
-                block.append("")
-                change_blocks.append("\n".join(block))
+                unchanged_str = ", ".join(
+                    f'{field}: "{value}"'
+                    for field, value in details["unchanged"].items()
+                )
+                block.append(f"    Unchanged: {unchanged_str}")
+            change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
         fragment = "{} rows added".format(len(result["added"]))
@@ -171,7 +169,7 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n\n".join(rows))
+        summary.append("\n".join(rows))
         summary.append("")
     if result["removed"]:
         fragment = "{} rows removed".format(len(result["removed"]))
@@ -184,16 +182,14 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n\n".join(rows))
+        summary.append("\n".join(rows))
         summary.append("")
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
 
 def human_row(row, prefix=""):
-    bits = []
-    for key, value in row.items():
-        bits.append("{}{}: {}".format(prefix, key, value))
-    return "\n".join(bits)
+    # Compact: key1=value1, key2=value2, ...
+    return prefix + ", ".join(f"{k}={v}" for k, v in row.items())
 
 
 def human_extras(row, extras):

From 47d52486da3a5620c2c0b2678d4b42660b26db79 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:45:37 -0500
Subject: [PATCH 039/129] fix

---
 db_diff/__init__.py | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index fbdeae0..151de43 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -141,22 +141,24 @@ def human_text(result, key=None, current=None, extras=None):
             summary.append(fragment + "\n")
         change_blocks = []
         for details in result["changed"]:
-            # Compact: key: field1: "old" => "new", field2: "old" => "new", ...
-            changes_str = ", ".join(
-                f'{field}: "{prev}" => "{curr}"'
-                for field, (prev, curr) in details["changes"].items()
-            )
-            block = [f"  {key}: {details['key']} | {changes_str}"]
+            block = []
+            block.append("  {}: {}".format(key, details["key"]))
+            for field, (prev_value, current_value) in details["changes"].items():
+                block.append(
+                    '    {}: "{}" => "{}"'.format(field, prev_value, current_value)
+                )
             if extras:
                 current_item = current[details["key"]]
                 block.append(human_extras(current_item, extras))
-            if details.get("unchanged"):
-                unchanged_str = ", ".join(
-                    f'{field}: "{value}"'
-                    for field, value in details["unchanged"].items()
-                )
-                block.append(f"    Unchanged: {unchanged_str}")
+            block.append("")
             change_blocks.append("\n".join(block))
+            if details.get("unchanged"):
+                block = []
+                block.append("    Unchanged:")
+                for field, value in details["unchanged"].items():
+                    block.append('      {}: "{}"'.format(field, value))
+                block.append("")
+                change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
         fragment = "{} rows added".format(len(result["added"]))
@@ -169,7 +171,7 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n".join(rows))
+        summary.append("\n\n".join(rows))
         summary.append("")
     if result["removed"]:
         fragment = "{} rows removed".format(len(result["removed"]))
@@ -182,14 +184,16 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n".join(rows))
+        summary.append("\n\n".join(rows))
         summary.append("")
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
 
 def human_row(row, prefix=""):
-    # Compact: key1=value1, key2=value2, ...
-    return prefix + ", ".join(f"{k}={v}" for k, v in row.items())
+    bits = []
+    for key, value in row.items():
+        bits.append("{}{}: {}".format(prefix, key, value))
+    return "\n".join(bits)
 
 
 def human_extras(row, extras):

From 456e2a5d9fbf5229cb5470bcb8fde326e129cc88 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:50:28 -0500
Subject: [PATCH 040/129] removed shorthand commands for key and output causing
 issues

---
 db_diff/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 95d4564..02c0bea 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -17,7 +17,7 @@
     type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
 )
 @click.option(
-    "-k", "--key", type=str, default=None, help="Column to use as a unique ID for each row (default: first column if not specified) (Example: --key=Id)"
+    "--key", type=str, default=None, help="Column to use as a unique ID for each row (default: first column if not specified) (Example: --key=Id)"
 )
 @click.option(
     "--format",
@@ -50,7 +50,7 @@
     help="Measure and display elapsed time for the diff operation",
 )
 @click.option(
-    "-o", "--output",
+    "--output",
     default="readable",
     show_default=True,
     help="Output format. Available: (readable|json|jsonfile) (default: readable)",

From ed68c6d3a8f1660afa60ec7e78d8b991ae7caf8d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:51:26 -0500
Subject: [PATCH 041/129] works now?

---
 db_diff/__init__.py | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 151de43..fbdeae0 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -141,24 +141,22 @@ def human_text(result, key=None, current=None, extras=None):
             summary.append(fragment + "\n")
         change_blocks = []
         for details in result["changed"]:
-            block = []
-            block.append("  {}: {}".format(key, details["key"]))
-            for field, (prev_value, current_value) in details["changes"].items():
-                block.append(
-                    '    {}: "{}" => "{}"'.format(field, prev_value, current_value)
-                )
+            # Compact: key: field1: "old" => "new", field2: "old" => "new", ...
+            changes_str = ", ".join(
+                f'{field}: "{prev}" => "{curr}"'
+                for field, (prev, curr) in details["changes"].items()
+            )
+            block = [f"  {key}: {details['key']} | {changes_str}"]
             if extras:
                 current_item = current[details["key"]]
                 block.append(human_extras(current_item, extras))
-            block.append("")
-            change_blocks.append("\n".join(block))
             if details.get("unchanged"):
-                block = []
-                block.append("    Unchanged:")
-                for field, value in details["unchanged"].items():
-                    block.append('      {}: "{}"'.format(field, value))
-                block.append("")
-                change_blocks.append("\n".join(block))
+                unchanged_str = ", ".join(
+                    f'{field}: "{value}"'
+                    for field, value in details["unchanged"].items()
+                )
+                block.append(f"    Unchanged: {unchanged_str}")
+            change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
         fragment = "{} rows added".format(len(result["added"]))
@@ -171,7 +169,7 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n\n".join(rows))
+        summary.append("\n".join(rows))
         summary.append("")
     if result["removed"]:
         fragment = "{} rows removed".format(len(result["removed"]))
@@ -184,16 +182,14 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n\n".join(rows))
+        summary.append("\n".join(rows))
         summary.append("")
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
 
 def human_row(row, prefix=""):
-    bits = []
-    for key, value in row.items():
-        bits.append("{}{}: {}".format(prefix, key, value))
-    return "\n".join(bits)
+    # Compact: key1=value1, key2=value2, ...
+    return prefix + ", ".join(f"{k}={v}" for k, v in row.items())
 
 
 def human_extras(row, extras):

From 88b4d0d3f25a94f20e5d6a390d794c2e53619f9d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 15:52:35 -0500
Subject: [PATCH 042/129] Update README.md

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index afb379e..61aa791 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,9 @@ Python CLI tool and library for comparing CSV database dumps and finding differe
 
     pip install git+https://github.com/datsom1/db-diff.git
 
+    To install unofficial latest version (you probably don't need to do this)
+    pip install --upgrade --force-reinstall git+https://github.com/datsom1/db-diff.git
+
 ## Usage
 
 Consider two CSV files:

From b41a3f1de35c5041f86a62ebd97cd210bd49777b Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:01:48 -0500
Subject: [PATCH 043/129] fixed bug where if --key not specified it would not
 set correctly

---
 db_diff/__init__.py | 36 ++++++++++++++++++++----------------
 db_diff/cli.py      | 35 +++++++++++++++++++++++++----------
 2 files changed, 45 insertions(+), 26 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index fbdeae0..151de43 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -141,22 +141,24 @@ def human_text(result, key=None, current=None, extras=None):
             summary.append(fragment + "\n")
         change_blocks = []
         for details in result["changed"]:
-            # Compact: key: field1: "old" => "new", field2: "old" => "new", ...
-            changes_str = ", ".join(
-                f'{field}: "{prev}" => "{curr}"'
-                for field, (prev, curr) in details["changes"].items()
-            )
-            block = [f"  {key}: {details['key']} | {changes_str}"]
+            block = []
+            block.append("  {}: {}".format(key, details["key"]))
+            for field, (prev_value, current_value) in details["changes"].items():
+                block.append(
+                    '    {}: "{}" => "{}"'.format(field, prev_value, current_value)
+                )
             if extras:
                 current_item = current[details["key"]]
                 block.append(human_extras(current_item, extras))
-            if details.get("unchanged"):
-                unchanged_str = ", ".join(
-                    f'{field}: "{value}"'
-                    for field, value in details["unchanged"].items()
-                )
-                block.append(f"    Unchanged: {unchanged_str}")
+            block.append("")
             change_blocks.append("\n".join(block))
+            if details.get("unchanged"):
+                block = []
+                block.append("    Unchanged:")
+                for field, value in details["unchanged"].items():
+                    block.append('      {}: "{}"'.format(field, value))
+                block.append("")
+                change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
         fragment = "{} rows added".format(len(result["added"]))
@@ -169,7 +171,7 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n".join(rows))
+        summary.append("\n\n".join(rows))
         summary.append("")
     if result["removed"]:
         fragment = "{} rows removed".format(len(result["removed"]))
@@ -182,14 +184,16 @@ def human_text(result, key=None, current=None, extras=None):
             if extras:
                 to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
-        summary.append("\n".join(rows))
+        summary.append("\n\n".join(rows))
         summary.append("")
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
 
 def human_row(row, prefix=""):
-    # Compact: key1=value1, key2=value2, ...
-    return prefix + ", ".join(f"{k}={v}" for k, v in row.items())
+    bits = []
+    for key, value in row.items():
+        bits.append("{}{}: {}".format(prefix, key, value))
+    return "\n".join(bits)
 
 
 def human_extras(row, extras):
diff --git a/db_diff/cli.py b/db_diff/cli.py
index 02c0bea..4889edd 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -85,7 +85,23 @@ def cli(
             ctx=click.get_current_context(),
         )
 
-    def load(filename):
+    def sniff_key_column(filename, input_format):
+        # Only for CSV/TSV/JSON
+        if input_format == "json":
+            with open(filename, encoding=encoding) as f:
+                import json
+                data = json.load(f)
+                if isinstance(data, list) and data:
+                    return list(data[0].keys())[0]
+        else:
+            with open(filename, newline="", encoding=encoding) as f:
+                import csv
+                reader = csv.reader(f, dialect=dialect.get(input_format))
+                headings = next(reader)
+                return headings[0]
+        return None
+
+    def load(filename, key):
         try:
             if input_format == "json":
                 with open(filename, encoding=encoding) as f:
@@ -98,17 +114,16 @@ def load(filename):
 
     if show_time:
         start = time_module.time()
-    # Load previous data first to determine key if not set
-    previous_data = load(previous)
-    # If key is not set, use the first column from previous_data
+
+    # Detect key column before loading data
     if key is None:
-        if previous_data and isinstance(previous_data, dict):
-            first_row = next(iter(previous_data.values()))
-            key = list(first_row.keys())[0]
-        else:
+        detected_format = input_format or previous.split(".")[-1].lower()
+        key = sniff_key_column(previous, detected_format)
+        if not key:
             raise click.ClickException("Could not determine key column from the previous file.")
-    # Now load current data with the determined key
-    current_data = load(current)
+
+    previous_data = load(previous, key)
+    current_data = load(current, key)
     diff = compare(previous_data, current_data, show_unchanged)
 
     if output == "json":

From bb27a291acaabf70ff8e8503f2846042b8434456 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:32:14 -0500
Subject: [PATCH 044/129] removed extra option and other fixes

---
 db_diff/cli.py | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 4889edd..bf80f7b 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -26,16 +26,9 @@
     help="Explicitly specify input format. Available (csv|tsv|json) (default: auto-detect based on file extension)",
 )
 @click.option(
-    "--show-unchanged",
+    "--showunchanged",
     is_flag=True,
-    help="Show unchanged fields for rows with at least one change",
-)
-@click.option(
-    "extras",
-    "--extra",
-    type=(str, str),
-    multiple=True,
-    help="key: format string - define extra fields to display",
+    help="If a record is changed, show ALL fields, not just the changed fields.",
 )
 @click.option(
     "--encoding",
@@ -65,7 +58,7 @@
 @click.version_option()
 def cli(
     previous, current, key, input_format,
-    show_unchanged, extras, encoding, show_time, output, output_file
+    show_unchanged, encoding, show_time, output, output_file
 ):
     """Compare the differences between two CSV or JSON files."""
     dialect = {
@@ -73,12 +66,6 @@ def cli(
         "tsv": "excel-tab",
     }
 
-    if extras and output in ("json", "jsonfile"):
-        raise click.UsageError(
-            "Extra fields are not supported in JSON output mode",
-            ctx=click.get_current_context(),
-        )
-
     if output == "jsonfile" and not output_file:
         raise click.UsageError(
             "--outputfile must be specified when --output=jsonfile",
@@ -133,7 +120,7 @@ def load(filename, key):
             f.write(std_json.dumps(diff, indent=4))
     else:
         print(
-            human_text(diff, key, current=current_data, extras=extras)
+            human_text(diff, key, current=current_data)
         )
 
     if show_time:

From fd7e60dfb250e63b489120a8f7b014a0985266ac Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:33:50 -0500
Subject: [PATCH 045/129] reordered options

---
 db_diff/cli.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index bf80f7b..ae5888a 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -16,20 +16,12 @@
     "current",
     type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
 )
-@click.option(
-    "--key", type=str, default=None, help="Column to use as a unique ID for each row (default: first column if not specified) (Example: --key=Id)"
-)
 @click.option(
     "--format",
     "input_format",
     default=None,
     help="Explicitly specify input format. Available (csv|tsv|json) (default: auto-detect based on file extension)",
 )
-@click.option(
-    "--showunchanged",
-    is_flag=True,
-    help="If a record is changed, show ALL fields, not just the changed fields.",
-)
 @click.option(
     "--encoding",
     type=str,
@@ -37,10 +29,10 @@
     help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) (default: utf-8)",
 )
 @click.option(
-    "--time",
-    "show_time",
-    is_flag=True,
-    help="Measure and display elapsed time for the diff operation",
+    "--key",
+    type=str,
+    default=None,
+    help="Column to use as a unique ID for each row (default: first column if not specified) (Example: --key=Id)",
 )
 @click.option(
     "--output",
@@ -55,6 +47,17 @@
     default=None,
     help="File to write JSON output to (only used with --output=jsonfile)",
 )
+@click.option(
+    "--showunchanged",
+    is_flag=True,
+    help="If a record is changed, show ALL fields, not just the changed fields.",
+)
+@click.option(
+    "--time",
+    "show_time",
+    is_flag=True,
+    help="Measure and display elapsed time for the diff operation",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,

From 411d6f0b8e552a6e76258439f79bbc0ab7a425ba Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:35:46 -0500
Subject: [PATCH 046/129] fixed helptext

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index ae5888a..6827e85 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -38,7 +38,7 @@
     "--output",
     default="readable",
     show_default=True,
-    help="Output format. Available: (readable|json|jsonfile) (default: readable)",
+    help="Output format. Available: (readable|json|jsonfile)",
 )
 @click.option(
     "--outputfile",

From ef75361ce630da61de0d6ed3f55997e8a80f6815 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:38:01 -0500
Subject: [PATCH 047/129] reordered options

---
 db_diff/cli.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 6827e85..a6f33d6 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -16,18 +16,6 @@
     "current",
     type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
 )
-@click.option(
-    "--format",
-    "input_format",
-    default=None,
-    help="Explicitly specify input format. Available (csv|tsv|json) (default: auto-detect based on file extension)",
-)
-@click.option(
-    "--encoding",
-    type=str,
-    default="utf-8",
-    help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) (default: utf-8)",
-)
 @click.option(
     "--key",
     type=str,
@@ -58,6 +46,18 @@
     is_flag=True,
     help="Measure and display elapsed time for the diff operation",
 )
+@click.option(
+    "--format",
+    "input_format",
+    default=None,
+    help="Explicitly specify input format. Available (csv|tsv|json) (default: auto-detect based on file extension)",
+)
+@click.option(
+    "--encoding",
+    type=str,
+    default="utf-8",
+    help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) (default: utf-8)",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,

From 08a7d247b25ab191cb27fc081bf6dd0ce8e1ec45 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:43:56 -0500
Subject: [PATCH 048/129] Update README.md

---
 README.md | 65 +++++++++++++++----------------------------------------
 1 file changed, 18 insertions(+), 47 deletions(-)

diff --git a/README.md b/README.md
index 61aa791..e77a5b3 100644
--- a/README.md
+++ b/README.md
@@ -18,19 +18,19 @@ Consider two CSV files:
 
 `one.csv`
 
-    id,name,age
+    Id,name,age
     1,Cleo,4
     2,Pancakes,2
 
 `two.csv`
 
-    id,name,age
+    Id,name,age
     1,Cleo,5
     3,Bailey,1
 
 `db-diff` can show a human-readable summary of differences between the files:
 
-    $ db-diff one.csv two.csv --key=id
+    $ db-diff one.csv two.csv --key=Id
     1 row changed, 1 row added, 1 row removed
 
     1 row changed
@@ -40,17 +40,17 @@ Consider two CSV files:
 
     1 row added
 
-      id: 3
+      Id: 3
       name: Bailey
       age: 1
 
     1 row removed
 
-      id: 2
+      Id: 2
       name: Pancakes
       age: 2
 
-The `--key=id` option means that the `id` column should be treated as the unique key, to identify which records have changed.
+The `--key=Id` option means that the `Id` column should be treated as the unique key, to identify which records have changed.
 
 The tool will automatically detect if your files are comma- or tab-separated. You can over-ride this automatic detection and force the tool to use a specific format using `--format=tsv` or `--format=csv`.
 
@@ -58,10 +58,10 @@ You can also feed it JSON files, provided they are a JSON array of objects where
 
 Use `--show-unchanged` to include full details of the unchanged values for rows with at least one change in the diff output:
 
-    % db-diff one.csv two.csv --key=id --show-unchanged
+    % db-diff one.csv two.csv --key=Id --showunchanged
     1 row changed
 
-      id: 1
+      Id: 1
         age: "4" => "5"
 
         Unchanged:
@@ -69,9 +69,9 @@ Use `--show-unchanged` to include full details of the unchanged values for rows
 
 ### JSON output
 
-You can use the `--json` option to get a machine-readable difference:
+You can use the `--output=json` option to get a machine-readable difference:
 
-    $ db-diff one.csv two.csv --key=id --json
+    $ db-diff one.csv two.csv --key=Id --output=json
     {
         "added": [
             {
@@ -102,28 +102,17 @@ You can use the `--json` option to get a machine-readable difference:
         "columns_removed": []
     }
 
-### Adding templated extras
+### JSON file output
 
-You can specify additional keys to be displayed in the human-readable format using the `--extra` option:
+You can use the `--output=jsonfile` and `--outputfile=` option to automatically save a .json file of the output:
 
-    --extra name "Python format string with {id} for variables"
+    $ db-diff one.csv two.csv --key=Id --output=jsonfile --outputfile=diffs.json
 
-For example, to output a link to `https://news.ycombinator.com/latest?id={id}` for each item with an ID, you could use this:
+### Measure time
 
-```bash
-db-diff one.csv two.csv --key=id \
-  --extra latest "https://news.ycombinator.com/latest?id={id}"
-```
-These extras display something like this:
-```
-1 row changed
+You can use the `--time` option to meaure the time it takes:
 
-  id: 41459472
-    points: "24" => "25"
-    numComments: "5" => "6"
-  extras:
-    latest: https://news.ycombinator.com/latest?id=41459472
-```
+    $ db-diff one.csv two.csv --key=Id --time
 
 ## As a Python library
 
@@ -131,28 +120,10 @@ You can also import the Python library into your own code like so:
 
     from csv_diff import load_csv, compare
     diff = compare(
-        load_csv(open("one.csv"), key="id"),
-        load_csv(open("two.csv"), key="id")
+        load_csv(open("one.csv"), key="Id"),
+        load_csv(open("two.csv"), key="Id")
     )
 
 `diff` will now contain the same data structure as the output in the `--json` example above.
 
 If the columns in the CSV have changed, those added or removed columns will be ignored when calculating changes made to specific rows.
-
-## As a Docker container
-
-### Build the image
-
-    $ docker build -t csvdiff .
-
-### Run the container
-
-    $ docker run --rm -v $(pwd):/files csvdiff
-
-Suppose current directory contains two csv files : one.csv two.csv
-
-    $ docker run --rm -v $(pwd):/files csvdiff one.csv two.csv
-    
-## Alternatives
-
-- [csvdiff](https://github.com/aswinkarthik/csvdiff) is a "fast diff tool for comparing CSV files" - you may get better results from this than from `db-diff` against larger files.

From 6dd58ec335dafff969e85e2ac0abb58025468739 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:48:33 -0500
Subject: [PATCH 049/129] Updated helptext

---
 db_diff/cli.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index a6f33d6..2515659 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -20,20 +20,20 @@
     "--key",
     type=str,
     default=None,
-    help="Column to use as a unique ID for each row (default: first column if not specified) (Example: --key=Id)",
+    help="Column to use as a unique ID for each row (ex: --key=Id). [default: first column if not specified]",
 )
 @click.option(
     "--output",
     default="readable",
     show_default=True,
-    help="Output format. Available: (readable|json|jsonfile)",
+    help="Output format. Available: (readable|json|jsonfile).",
 )
 @click.option(
     "--outputfile",
     "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
-    help="File to write JSON output to (only used with --output=jsonfile)",
+    help="File to write JSON output to (only used with --output=jsonfile).",
 )
 @click.option(
     "--showunchanged",
@@ -44,19 +44,20 @@
     "--time",
     "show_time",
     is_flag=True,
-    help="Measure and display elapsed time for the diff operation",
+    help="Measure and display elapsed time for the diff operation.",
 )
 @click.option(
     "--format",
     "input_format",
     default=None,
-    help="Explicitly specify input format. Available (csv|tsv|json) (default: auto-detect based on file extension)",
+    help="Explicitly specify input format. Available (csv|tsv|json). [default: auto-detect based on file extension]",
 )
 @click.option(
     "--encoding",
     type=str,
     default="utf-8",
-    help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) (default: utf-8)",
+    show_default=True,
+    help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).",
 )
 @click.version_option()
 def cli(

From 92ad55b82d89672910ffff266fdb3866e7826c8d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:49:21 -0500
Subject: [PATCH 050/129] Update README.md

---
 README.md | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/README.md b/README.md
index e77a5b3..6193905 100644
--- a/README.md
+++ b/README.md
@@ -127,3 +127,22 @@ You can also import the Python library into your own code like so:
 `diff` will now contain the same data structure as the output in the `--json` example above.
 
 If the columns in the CSV have changed, those added or removed columns will be ignored when calculating changes made to specific rows.
+
+## Full CLI Helptext
+
+`Usage: db-diff \[OPTIONS\] PREVIOUS CURRENT
+
+  Compare the differences between two CSV or JSON files to find differences.
+
+Options:
+  --format TEXT      Explicitly specify input format. Available (csv|tsv|json) \[default: auto-detect based on file extension\]
+  --encoding TEXT    Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) \[default: utf-8\]
+  --key TEXT         Column to use as a unique ID for each row (ex: --key=Id) \[default: first column if not specified\] 
+  --output TEXT      Output format. Available: (readable|json|jsonfile)  \[default: readable\]
+  --outputfile FILE  File to write JSON output to (only used with --output=jsonfile)
+  --showunchanged    If a record is changed, show ALL fields, not just the changed fields.
+  --time             Measure and display elapsed time for the diff operation
+  --version          Show the version and exit.
+  -h, --help         Show this message and exit.
+
+  Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json`

From eee8ea5d1fc222e8198bf9a3bd21be9515a4ecca Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:50:36 -0500
Subject: [PATCH 051/129] Update README.md

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 6193905..ea13f59 100644
--- a/README.md
+++ b/README.md
@@ -130,7 +130,8 @@ If the columns in the CSV have changed, those added or removed columns will be i
 
 ## Full CLI Helptext
 
-`Usage: db-diff \[OPTIONS\] PREVIOUS CURRENT
+$ db-diff --help
+Usage: db-diff \[OPTIONS\] PREVIOUS CURRENT
 
   Compare the differences between two CSV or JSON files to find differences.
 
@@ -145,4 +146,4 @@ Options:
   --version          Show the version and exit.
   -h, --help         Show this message and exit.
 
-  Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json`
+  Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json

From f4a7f30b19b90ffbdd20f4f9b4562d1819bc2b7c Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:53:39 -0500
Subject: [PATCH 052/129] Update README.md

---
 README.md | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index ea13f59..a4ecc51 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,8 @@ Python CLI tool and library for comparing CSV database dumps and finding differe
 
     pip install git+https://github.com/datsom1/db-diff.git
 
-    To install unofficial latest version (you probably don't need to do this)
+To install the unofficial latest version (you probably don't need to do this):
+
     pip install --upgrade --force-reinstall git+https://github.com/datsom1/db-diff.git
 
 ## Usage
@@ -130,20 +131,20 @@ If the columns in the CSV have changed, those added or removed columns will be i
 
 ## Full CLI Helptext
 
-$ db-diff --help
-Usage: db-diff \[OPTIONS\] PREVIOUS CURRENT
-
-  Compare the differences between two CSV or JSON files to find differences.
-
-Options:
-  --format TEXT      Explicitly specify input format. Available (csv|tsv|json) \[default: auto-detect based on file extension\]
-  --encoding TEXT    Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) \[default: utf-8\]
-  --key TEXT         Column to use as a unique ID for each row (ex: --key=Id) \[default: first column if not specified\] 
-  --output TEXT      Output format. Available: (readable|json|jsonfile)  \[default: readable\]
-  --outputfile FILE  File to write JSON output to (only used with --output=jsonfile)
-  --showunchanged    If a record is changed, show ALL fields, not just the changed fields.
-  --time             Measure and display elapsed time for the diff operation
-  --version          Show the version and exit.
-  -h, --help         Show this message and exit.
-
-  Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json
+    $ db-diff --help
+    Usage: db-diff \[OPTIONS\] PREVIOUS CURRENT
+    
+      Compare the differences between two CSV or JSON files to find differences.
+    
+    Options:
+      --format TEXT      Explicitly specify input format. Available (csv|tsv|json) \[default: auto-detect based on file extension\]
+      --encoding TEXT    Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) \[default: utf-8\]
+      --key TEXT         Column to use as a unique ID for each row (ex: --key=Id) \[default: first column if not specified\] 
+      --output TEXT      Output format. Available: (readable|json|jsonfile)  \[default: readable\]
+      --outputfile FILE  File to write JSON output to (only used with --output=jsonfile)
+      --showunchanged    If a record is changed, show ALL fields, not just the changed fields.
+      --time             Measure and display elapsed time for the diff operation
+      --version          Show the version and exit.
+      -h, --help         Show this message and exit.
+    
+      Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json

From d7169de5744cdd7e7b1759f65b1b04bb7efc8853 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:54:51 -0500
Subject: [PATCH 053/129] Update README.md

Updated helptext
---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index a4ecc51..2c20705 100644
--- a/README.md
+++ b/README.md
@@ -132,15 +132,15 @@ If the columns in the CSV have changed, those added or removed columns will be i
 ## Full CLI Helptext
 
     $ db-diff --help
-    Usage: db-diff \[OPTIONS\] PREVIOUS CURRENT
+    Usage: db-diff [OPTIONS] PREVIOUS CURRENT
     
       Compare the differences between two CSV or JSON files to find differences.
     
     Options:
-      --format TEXT      Explicitly specify input format. Available (csv|tsv|json) \[default: auto-detect based on file extension\]
-      --encoding TEXT    Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) \[default: utf-8\]
-      --key TEXT         Column to use as a unique ID for each row (ex: --key=Id) \[default: first column if not specified\] 
-      --output TEXT      Output format. Available: (readable|json|jsonfile)  \[default: readable\]
+      --format TEXT      Explicitly specify input format. Available (csv|tsv|json) [default: auto-detect based on file extension]
+      --encoding TEXT    Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) [default: utf-8]
+      --key TEXT         Column to use as a unique ID for each row (ex: --key=Id) [default: first column if not specified] 
+      --output TEXT      Output format. Available: (readable|json|jsonfile)  [default: readable]
       --outputfile FILE  File to write JSON output to (only used with --output=jsonfile)
       --showunchanged    If a record is changed, show ALL fields, not just the changed fields.
       --time             Measure and display elapsed time for the diff operation

From c8338527a5769c1c9089b4ea2fad00d75721553d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:57:10 -0500
Subject: [PATCH 054/129] fixed error

---
 db_diff/cli.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 2515659..96c96b4 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -37,6 +37,7 @@
 )
 @click.option(
     "--showunchanged",
+    "show_unchanged",
     is_flag=True,
     help="If a record is changed, show ALL fields, not just the changed fields.",
 )

From 31f628f296e3f6c5f59e753a893722ff2bea0f01 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:57:22 -0500
Subject: [PATCH 055/129] Update README.md

typo
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2c20705..3528008 100644
--- a/README.md
+++ b/README.md
@@ -125,7 +125,7 @@ You can also import the Python library into your own code like so:
         load_csv(open("two.csv"), key="Id")
     )
 
-`diff` will now contain the same data structure as the output in the `--json` example above.
+`diff` will now contain the same data structure as the output in the `--output=json` example above.
 
 If the columns in the CSV have changed, those added or removed columns will be ignored when calculating changes made to specific rows.
 

From 1fadd2a6e43784a15da11a1370c084050c625dc5 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 2 Jun 2025 17:00:14 -0500
Subject: [PATCH 056/129] Update README.md

---
 README.md | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 3528008..bebd2bd 100644
--- a/README.md
+++ b/README.md
@@ -32,20 +32,20 @@ Consider two CSV files:
 `db-diff` can show a human-readable summary of differences between the files:
 
     $ db-diff one.csv two.csv --key=Id
-    1 row changed, 1 row added, 1 row removed
+    1 rows changed, 1 rows added, 1 rows removed
 
-    1 row changed
+    1 rows changed
 
-      Row 1
+      Rows 1
         age: "4" => "5"
 
-    1 row added
+    1 rows added
 
       Id: 3
       name: Bailey
       age: 1
 
-    1 row removed
+    1 rows removed
 
       Id: 2
       name: Pancakes
@@ -60,7 +60,7 @@ You can also feed it JSON files, provided they are a JSON array of objects where
 Use `--show-unchanged` to include full details of the unchanged values for rows with at least one change in the diff output:
 
     % db-diff one.csv two.csv --key=Id --showunchanged
-    1 row changed
+    1 rows changed
 
       Id: 1
         age: "4" => "5"
@@ -114,6 +114,10 @@ You can use the `--output=jsonfile` and `--outputfile=` option to automatically
 You can use the `--time` option to meaure the time it takes:
 
     $ db-diff one.csv two.csv --key=Id --time
+    .
+    .
+    .
+    Elapsed time: 0.016 seconds
 
 ## As a Python library
 

From 1a64b902e7c9444c85badf0342e33702b90f9e7a Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 3 Jun 2025 10:17:51 -0500
Subject: [PATCH 057/129] can now specify output filepath and output filename
 defaults to time

---
 db_diff/cli.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 96c96b4..c8a3185 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -1,12 +1,14 @@
 import click
 import json as std_json
 import time as time_module
+import os
+import datetime
 from . import load_csv, load_json, compare, human_text
 
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
     help="Compare the differences between two CSV or JSON files to find differences.",
-    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json",
+    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outfilename=diff.json",
 )
 @click.argument(
     "previous",
@@ -29,12 +31,19 @@
     help="Output format. Available: (readable|json|jsonfile).",
 )
 @click.option(
-    "--outputfile",
+    "--outfilename",
     "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
     help="File to write JSON output to (only used with --output=jsonfile).",
 )
+@click.option(
+    "--outfilepath",
+    "output_path",
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+    default=None,
+    help="Directory to save the output file (only used with --output=jsonfile).",
+)
 @click.option(
     "--showunchanged",
     "show_unchanged",
@@ -63,7 +72,7 @@
 @click.version_option()
 def cli(
     previous, current, key, input_format,
-    show_unchanged, encoding, show_time, output, output_file
+    show_unchanged, encoding, show_time, output, output_file, output_path
 ):
     """Compare the differences between two CSV or JSON files."""
     dialect = {
@@ -71,11 +80,15 @@ def cli(
         "tsv": "excel-tab",
     }
 
-    if output == "jsonfile" and not output_file:
-        raise click.UsageError(
-            "--outputfile must be specified when --output=jsonfile",
-            ctx=click.get_current_context(),
-        )
+    # Set default output filename if needed
+    if output == "jsonfile":
+        if not output_file:
+            timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+            output_file = f"{timestamp}_diffs.json"
+        if output_path:
+            if not os.path.isdir(output_path):
+                raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
+            output_file = os.path.join(output_path, os.path.basename(output_file))
 
     def sniff_key_column(filename, input_format):
         # Only for CSV/TSV/JSON

From d5395a87383e8ea5315af3434adeaf28efe01c8a Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 3 Jun 2025 10:27:01 -0500
Subject: [PATCH 058/129] added default helptext to options

---
 db_diff/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index c8a3185..44c5f2a 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -35,14 +35,14 @@
     "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
-    help="File to write JSON output to (only used with --output=jsonfile).",
+    help="File to write JSON output to (only used with --output=jsonfile) [default: YYYY-MM-DD_HH-MM-SS_diffs.json].",
 )
 @click.option(
     "--outfilepath",
     "output_path",
     type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
     default=None,
-    help="Directory to save the output file (only used with --output=jsonfile).",
+    help="Directory to save the output file (only used with --output=jsonfile) [default: current directory].",
 )
 @click.option(
     "--showunchanged",

From 6bae40adba230103d75fb50e9a21c5586cd4848e Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 3 Jun 2025 10:28:32 -0500
Subject: [PATCH 059/129] fixes

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 44c5f2a..2cbc6a5 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -22,7 +22,7 @@
     "--key",
     type=str,
     default=None,
-    help="Column to use as a unique ID for each row (ex: --key=Id). [default: first column if not specified]",
+    help="Column to use as a unique ID for each row (ex: --key=Id). [default: first column header]",
 )
 @click.option(
     "--output",

From 5ab7878ab1f8fecbcf6b6f434076cfee1b020926 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 3 Jun 2025 11:08:19 -0500
Subject: [PATCH 060/129] Added --fields and --ignorefields options allowing
 specification of which fields will be compared

---
 db_diff/__init__.py | 43 +++++++++++++++++++++++++++++--------------
 db_diff/cli.py      | 31 +++++++++++++++++++++++++++++--
 2 files changed, 58 insertions(+), 16 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 151de43..d434642 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -52,7 +52,7 @@ def _simplify_json_row(r, common_keys):
     return r
 
 
-def compare(previous, current, show_unchanged=False):
+def compare(previous, current, show_unchanged=False, fields=None, ignorefields=None):
     result = {
         "added": [],
         "removed": [],
@@ -64,45 +64,60 @@ def compare(previous, current, show_unchanged=False):
     previous_columns = set(next(iter(previous.values())).keys())
     current_columns = set(next(iter(current.values())).keys())
     ignore_columns = None
-    if previous_columns != current_columns:
-        result["columns_added"] = [
-            c for c in current_columns if c not in previous_columns
-        ]
-        result["columns_removed"] = [
-            c for c in previous_columns if c not in current_columns
-        ]
-        ignore_columns = current_columns.symmetric_difference(previous_columns)
+
+    # Apply fields/ignorefields filtering
+    if fields:
+        compare_columns = set(fields)
+    elif ignorefields:
+        compare_columns = previous_columns | current_columns
+        compare_columns = compare_columns - set(ignorefields)
+    else:
+        compare_columns = previous_columns | current_columns
+
+    # Adjust columns_added/removed based on compare_columns
+    result["columns_added"] = [c for c in current_columns if c not in previous_columns and c in compare_columns]
+    result["columns_removed"] = [c for c in previous_columns if c not in current_columns and c in compare_columns]
+    ignore_columns = (previous_columns | current_columns) - compare_columns
+
     # Have any rows been removed or added?
     removed = [id for id in previous if id not in current]
     added = [id for id in current if id not in previous]
-    # How about changed?
     removed_or_added = set(removed) | set(added)
     potential_changes = [id for id in current if id not in removed_or_added]
-    changed = [id for id in potential_changes if current[id] != previous[id]]
+    changed = [
+        id for id in potential_changes
+        if any(
+            (k in compare_columns) and (current[id].get(k) != previous[id].get(k))
+            for k in compare_columns
+        )
+    ]
     if added:
         result["added"] = [current[id] for id in added]
     if removed:
         result["removed"] = [previous[id] for id in removed]
     if changed:
         for id in changed:
-            diffs = list(diff(previous[id], current[id], ignore=ignore_columns))
+            diffs = list(diff(
+                previous[id], current[id],
+                ignore=ignore_columns if ignore_columns else None
+            ))
             if diffs:
                 changes = {
                     "key": id,
                     "changes": {
-                        # field can be a list if id contained '.' - #7
                         field[0] if isinstance(field, list) else field: [
                             prev_value,
                             current_value,
                         ]
                         for _, field, (prev_value, current_value) in diffs
+                        if (field[0] if isinstance(field, list) else field) in compare_columns
                     },
                 }
                 if show_unchanged:
                     changes["unchanged"] = {
                         field: value
                         for field, value in previous[id].items()
-                        if field not in changes["changes"] and field != "id"
+                        if field not in changes["changes"] and field != "id" and field in compare_columns
                     }
                 result["changed"].append(changes)
     return result
diff --git a/db_diff/cli.py b/db_diff/cli.py
index 2cbc6a5..9e3fb6e 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -69,10 +69,23 @@
     show_default=True,
     help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).",
 )
+@click.option(
+    "--fields",
+    type=str,
+    default=None,
+    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields.",
+)
+@click.option(
+    "--ignorefields",
+    type=str,
+    default=None,
+    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields.",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,
-    show_unchanged, encoding, show_time, output, output_file, output_path
+    show_unchanged, encoding, show_time, output, output_file, output_path,
+    fields, ignorefields
 ):
     """Compare the differences between two CSV or JSON files."""
     dialect = {
@@ -127,9 +140,23 @@ def load(filename, key):
         if not key:
             raise click.ClickException("Could not determine key column from the previous file.")
 
+    # Enforce mutual exclusivity
+    if fields and ignorefields:
+        raise click.ClickException("You cannot use --fields and --ignorefields at the same time.")
+
+    # Parse fields/ignorefields into sets
+    fields_set = set(f.strip() for f in fields.split(",")) if fields else None
+    ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
+
     previous_data = load(previous, key)
     current_data = load(current, key)
-    diff = compare(previous_data, current_data, show_unchanged)
+    diff = compare(
+        previous_data,
+        current_data,
+        show_unchanged,
+        fields=fields_set,
+        ignorefields=ignorefields_set,
+    )
 
     if output == "json":
         print(std_json.dumps(diff, indent=4))

From 7ac8c4b645d4f7f27671bbaa226235199c27091b Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 3 Jun 2025 11:14:10 -0500
Subject: [PATCH 061/129] Updated helptext

---
 db_diff/cli.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 9e3fb6e..58db770 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -35,14 +35,14 @@
     "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
-    help="File to write JSON output to (only used with --output=jsonfile) [default: YYYY-MM-DD_HH-MM-SS_diffs.json].",
+    help="File to write JSON output to (only used with --output=jsonfile). [default: YYYY-MM-DD_HH-MM-SS_diffs.json]",
 )
 @click.option(
     "--outfilepath",
     "output_path",
     type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
     default=None,
-    help="Directory to save the output file (only used with --output=jsonfile) [default: current directory].",
+    help="Directory to save the output file (only used with --output=jsonfile). [default: current directory]",
 )
 @click.option(
     "--showunchanged",
@@ -73,13 +73,13 @@
     "--fields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields.",
+    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age__c,Email__c).",
 )
 @click.option(
     "--ignorefields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields.",
+    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp,Days_Since_Last_Contact__c).",
 )
 @click.version_option()
 def cli(

From ac92443e0f2c5cf062427a33ba3410f0ebf32a09 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 3 Jun 2025 11:14:57 -0500
Subject: [PATCH 062/129] reordered options

---
 db_diff/cli.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 58db770..84934ec 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -44,6 +44,18 @@
     default=None,
     help="Directory to save the output file (only used with --output=jsonfile). [default: current directory]",
 )
+@click.option(
+    "--fields",
+    type=str,
+    default=None,
+    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age__c,Email__c).",
+)
+@click.option(
+    "--ignorefields",
+    type=str,
+    default=None,
+    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp,Days_Since_Last_Contact__c).",
+)
 @click.option(
     "--showunchanged",
     "show_unchanged",
@@ -69,18 +81,6 @@
     show_default=True,
     help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).",
 )
-@click.option(
-    "--fields",
-    type=str,
-    default=None,
-    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age__c,Email__c).",
-)
-@click.option(
-    "--ignorefields",
-    type=str,
-    default=None,
-    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp,Days_Since_Last_Contact__c).",
-)
 @click.version_option()
 def cli(
     previous, current, key, input_format,

From bad123622ed95ba9e35a2025a4623ce3e1c9dd37 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 3 Jun 2025 13:36:31 -0500
Subject: [PATCH 063/129] updated version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 786c861..4572de4 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "2.7"
+VERSION = "3.1"
 
 
 def get_long_description():

From 2ab8e12b8c7427723c88d9cf946510e5d6f5e605 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Wed, 4 Jun 2025 14:27:12 -0500
Subject: [PATCH 064/129] test edit

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 84934ec..f792811 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -157,7 +157,7 @@ def load(filename, key):
         fields=fields_set,
         ignorefields=ignorefields_set,
     )
-
+ # test edit
     if output == "json":
         print(std_json.dumps(diff, indent=4))
     elif output == "jsonfile":

From e6de343f4b46b6314cd52a76cf2616d34458a148 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 15:49:06 -0500
Subject: [PATCH 065/129] Added --Streaming option for very large csv files

---
 db_diff/__init__.py | 70 +++++++++++++++++++++++++++++++++++++++++++++
 db_diff/cli.py      | 47 ++++++++++++++++++++++--------
 2 files changed, 105 insertions(+), 12 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index d434642..6ce979a 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -123,6 +123,76 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
     return result
 
 
+def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignorefields=None, encoding='utf-8', dialect='excel'):
+    """
+    Compare two sorted CSV files by streaming, returning a diff dict.
+    """
+    import csv
+    result = {
+        "added": [],
+        "removed": [],
+        "changed": [],
+        "columns_added": [],
+        "columns_removed": [],
+    }
+
+    with open(prev_path, newline='', encoding=encoding) as f1, open(curr_path, newline='', encoding=encoding) as f2:
+        reader1 = csv.DictReader(f1, dialect=dialect)
+        reader2 = csv.DictReader(f2, dialect=dialect)
+        prev_row = next(reader1, None)
+        curr_row = next(reader2, None)
+
+        prev_columns = set(reader1.fieldnames)
+        curr_columns = set(reader2.fieldnames)
+
+        # Determine columns to compare
+        if compare_columns:
+            compare_columns = set(compare_columns)
+        elif ignorefields:
+            compare_columns = (prev_columns | curr_columns) - set(ignorefields)
+        else:
+            compare_columns = prev_columns | curr_columns
+
+        result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns]
+        result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns]
+
+        while prev_row or curr_row:
+            if prev_row and curr_row:
+                if key not in prev_row or key not in curr_row:
+                    raise KeyError(f"Key column '{key}' missing in one of the rows.")
+                prev_key = str(prev_row[key])
+                curr_key = str(curr_row[key])
+                if prev_key == curr_key:
+                    # Check for changes
+                    changed_fields = {
+                        col: [prev_row.get(col), curr_row.get(col)]
+                        for col in compare_columns
+                        if prev_row.get(col) != curr_row.get(col)
+                    }
+                    if changed_fields:
+                        result["changed"].append({
+                            "key": prev_key,
+                            "changes": changed_fields
+                        })
+                    prev_row = next(reader1, None)
+                    curr_row = next(reader2, None)
+                elif prev_key < curr_key:
+                    # Row removed
+                    result["removed"].append(prev_row)
+                    prev_row = next(reader1, None)
+                else:
+                    # Row added
+                    result["added"].append(curr_row)
+                    curr_row = next(reader2, None)
+            elif prev_row:
+                result["removed"].append(prev_row)
+                prev_row = next(reader1, None)
+            elif curr_row:
+                result["added"].append(curr_row)
+                curr_row = next(reader2, None)
+    return result
+
+
 def human_text(result, key=None, current=None, extras=None):
     title = []
     summary = []
diff --git a/db_diff/cli.py b/db_diff/cli.py
index f792811..f5bc6b9 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -3,7 +3,7 @@
 import time as time_module
 import os
 import datetime
-from . import load_csv, load_json, compare, human_text
+from . import load_csv, load_json, compare, human_text, streaming_compare_csv
 
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
@@ -81,11 +81,16 @@
     show_default=True,
     help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).",
 )
+@click.option(
+    "--streaming",
+    is_flag=True,
+    help="Use streaming mode for very large CSV/TSV files (requires files to be sorted by key).",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,
     show_unchanged, encoding, show_time, output, output_file, output_path,
-    fields, ignorefields
+    fields, ignorefields, streaming
 ):
     """Compare the differences between two CSV or JSON files."""
     dialect = {
@@ -148,16 +153,34 @@ def load(filename, key):
     fields_set = set(f.strip() for f in fields.split(",")) if fields else None
     ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
 
-    previous_data = load(previous, key)
-    current_data = load(current, key)
-    diff = compare(
-        previous_data,
-        current_data,
-        show_unchanged,
-        fields=fields_set,
-        ignorefields=ignorefields_set,
-    )
- # test edit
+    # --- Streaming logic ---
+    if streaming and (input_format in ("csv", "tsv", None)):
+        # Default to csv if not specified
+        fmt = input_format or previous.split(".")[-1].lower()
+        if fmt not in ("csv", "tsv"):
+            raise click.ClickException("--streaming only works with CSV/TSV files.")
+        diff = streaming_compare_csv(
+            previous,
+            current,
+            key=key,
+            compare_columns=fields_set,
+            ignorefields=ignorefields_set,
+            encoding=encoding,
+            dialect=dialect.get(fmt, "excel"),
+        )
+        # For human_text, we need current_data for extras (if used)
+        current_data = None
+    else:
+        previous_data = load(previous, key)
+        current_data = load(current, key)
+        diff = compare(
+            previous_data,
+            current_data,
+            show_unchanged,
+            fields=fields_set,
+            ignorefields=ignorefields_set,
+        )
+
     if output == "json":
         print(std_json.dumps(diff, indent=4))
     elif output == "jsonfile":

From a9f957077f52d3f36c63e42c240195fbdb08f959 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 15:53:16 -0500
Subject: [PATCH 066/129] Update README.md

---
 README.md | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index bebd2bd..e34b480 100644
--- a/README.md
+++ b/README.md
@@ -141,14 +141,18 @@ If the columns in the CSV have changed, those added or removed columns will be i
       Compare the differences between two CSV or JSON files to find differences.
     
     Options:
-      --format TEXT      Explicitly specify input format. Available (csv|tsv|json) [default: auto-detect based on file extension]
-      --encoding TEXT    Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...) [default: utf-8]
-      --key TEXT         Column to use as a unique ID for each row (ex: --key=Id) [default: first column if not specified] 
-      --output TEXT      Output format. Available: (readable|json|jsonfile)  [default: readable]
-      --outputfile FILE  File to write JSON output to (only used with --output=jsonfile)
-      --showunchanged    If a record is changed, show ALL fields, not just the changed fields.
-      --time             Measure and display elapsed time for the diff operation
-      --version          Show the version and exit.
-      -h, --help         Show this message and exit.
+      --key TEXT               Column to use as a unique ID for each row (ex: --key=Id). [default: first column header]
+      --output TEXT            Output format. Available: (readable|json|jsonfile). [default: readable]
+      --outfilename FILE       File to write JSON output to (only used with --output=jsonfile). [default: YYYY-MM-DD_HH-MM-SS_diffs.json]
+      --outfilepath DIRECTORY  Directory to save the output file (only used with --output=jsonfile). [default: current directory]
+      --fields TEXT            Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age__c,Email__c).
+      --ignorefields TEXT      Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp,Days_Since_Last_Contact__c).
+      --showunchanged          If a record is changed, show ALL fields, not just the changed fields.
+      --time                   Measure and display elapsed time for the diff operation.
+      --format TEXT            Explicitly specify input format. Available (csv|tsv|json). [default: auto-detect based on file extension]
+      --encoding TEXT          Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).  [default: utf-8]
+      --streaming              Use streaming mode for very large CSV/TSV files (requires files to be sorted by key).
+      --version                Show the version and exit.
+      -h, --help               Show this message and exit.
     
-      Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outputfile=diff.json
+      Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outfilename=diff.json

From 8239932fe7a8c70123dc81e457f381f2bd088a62 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 15:53:32 -0500
Subject: [PATCH 067/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4572de4..2b92f11 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "3.1"
+VERSION = "3.2"
 
 
 def get_long_description():

From c7df95c775c36971c1ee683f3be7304da36fa49b Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 15:58:33 -0500
Subject: [PATCH 068/129] Added terminal output so you know that the program is
 working when running and not just frozen

---
 db_diff/cli.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index f5bc6b9..ba3847f 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -98,6 +98,8 @@ def cli(
         "tsv": "excel-tab",
     }
 
+    click.echo(f"Starting diff: '{previous}' vs '{current}'...")
+
     # Set default output filename if needed
     if output == "jsonfile":
         if not output_file:
@@ -183,13 +185,16 @@ def load(filename, key):
 
     if output == "json":
         print(std_json.dumps(diff, indent=4))
+        click.echo("Finished. Output printed as JSON to terminal.")
     elif output == "jsonfile":
         with open(output_file, "w", encoding=encoding) as f:
             f.write(std_json.dumps(diff, indent=4))
+        click.echo(f"Finished. Output saved to file: {output_file}")
     else:
         print(
             human_text(diff, key, current=current_data)
         )
+        click.echo("Finished. Output printed in readable format to terminal.")
 
     if show_time:
         end = time_module.time()

From 2c49458af49e2478550edca3a19eb07410bcd42d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 16:11:05 -0500
Subject: [PATCH 069/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2b92f11..64edec4 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "3.2"
+VERSION = "3.3"
 
 
 def get_long_description():

From 289bd525cbf18872db02173c493d6816b42e6584 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 16:14:44 -0500
Subject: [PATCH 070/129] Removed dockerfile

---
 Dockerfile | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 Dockerfile

diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 8749e91..0000000
--- a/Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM python:3.13-alpine
-
-# Copy your local code into the container
-COPY . /app
-WORKDIR /app
-
-# Install your package locally
-RUN pip install .
-
-WORKDIR /files
-ENTRYPOINT ["db-diff"]
-CMD ["--help"]

From a80ac5b9428c3f58baa1779cd091442829908b31 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 16:45:40 -0500
Subject: [PATCH 071/129] Updated hashing algorithm, made streaming more
 efficient (hopefully), also refactored lots of code (hopefully nothing is
 broken)

---
 db_diff/__init__.py | 104 +++++++++++++++++------------------
 db_diff/cli.py      | 129 ++++++++++++++++++++++++++++----------------
 2 files changed, 131 insertions(+), 102 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 6ce979a..72f89d7 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -1,10 +1,12 @@
 import csv
-from dictdiffer import diff
 import json
-import hashlib
-
+import xxhash
+from dictdiffer import diff
 
 def load_csv(fp, key=None, dialect=None):
+    """
+    Load a CSV file into a dictionary keyed by the given column or hash.
+    """
     if dialect is None and fp.seekable():
         # Peek at first 1MB to sniff the delimiter and other dialect details
         peek = fp.read(1024**2)
@@ -14,35 +16,35 @@ def load_csv(fp, key=None, dialect=None):
         except csv.Error:
             # Oh well, we tried. Fallback to the default.
             pass
-    fp = csv.reader(fp, dialect=(dialect or "excel"))
-    headings = next(fp)
-    rows = [dict(zip(headings, line)) for line in fp]
+    reader = csv.reader(fp, dialect=(dialect or "excel"))
+    headings = next(reader)
+    rows = [dict(zip(headings, line)) for line in reader]
     if key:
         keyfn = lambda r: r[key]
     else:
-        keyfn = lambda r: hashlib.sha1(
-            json.dumps(r, sort_keys=True).encode("utf8")
-        ).hexdigest()
+        keyfn = lambda r: xxhash.xxh64(json.dumps(r, sort_keys=True).encode("utf8")).hexdigest()
     return {keyfn(r): r for r in rows}
 
-
 def load_json(fp, key=None):
+    """
+    Load a JSON array of objects into a dictionary keyed by the given column or hash.
+    """
     raw_list = json.load(fp)
-    assert isinstance(raw_list, list)
+    if not isinstance(raw_list, list):
+        raise ValueError("JSON file must contain a list of objects.")
     common_keys = set()
     for item in raw_list:
         common_keys.update(item.keys())
     if key:
         keyfn = lambda r: r[key]
     else:
-        keyfn = lambda r: hashlib.sha1(
-            json.dumps(r, sort_keys=True).encode("utf8")
-        ).hexdigest()
+        keyfn = lambda r: xxhash.xxh64(json.dumps(r, sort_keys=True).encode("utf8")).hexdigest()
     return {keyfn(r): _simplify_json_row(r, common_keys) for r in raw_list}
 
-
 def _simplify_json_row(r, common_keys):
-    # Convert list/dict values into JSON serialized strings
+    """
+    Ensure all rows have the same keys and serialize nested structures.
+    """
     for key, value in r.items():
         if isinstance(value, (dict, tuple, list)):
             r[key] = json.dumps(value)
@@ -51,8 +53,10 @@ def _simplify_json_row(r, common_keys):
             r[key] = None
     return r
 
-
 def compare(previous, current, show_unchanged=False, fields=None, ignorefields=None):
+    """
+    Compare two dictionaries of rows and return a diff summary.
+    """
     result = {
         "added": [],
         "removed": [],
@@ -63,14 +67,13 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
     # Have the columns changed?
     previous_columns = set(next(iter(previous.values())).keys())
     current_columns = set(next(iter(current.values())).keys())
-    ignore_columns = None
 
     # Apply fields/ignorefields filtering
     if fields:
         compare_columns = set(fields)
     elif ignorefields:
         compare_columns = previous_columns | current_columns
-        compare_columns = compare_columns - set(ignorefields)
+        compare_columns -= set(ignorefields)
     else:
         compare_columns = previous_columns | current_columns
 
@@ -122,12 +125,10 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
                 result["changed"].append(changes)
     return result
 
-
 def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignorefields=None, encoding='utf-8', dialect='excel'):
     """
-    Compare two sorted CSV files by streaming, returning a diff dict.
+    Efficiently compare two sorted CSV files by streaming, returning a diff dict.
     """
-    import csv
     result = {
         "added": [],
         "removed": [],
@@ -135,7 +136,6 @@ def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignor
         "columns_added": [],
         "columns_removed": [],
     }
-
     with open(prev_path, newline='', encoding=encoding) as f1, open(curr_path, newline='', encoding=encoding) as f2:
         reader1 = csv.DictReader(f1, dialect=dialect)
         reader2 = csv.DictReader(f2, dialect=dialect)
@@ -158,10 +158,8 @@ def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignor
 
         while prev_row or curr_row:
             if prev_row and curr_row:
-                if key not in prev_row or key not in curr_row:
-                    raise KeyError(f"Key column '{key}' missing in one of the rows.")
-                prev_key = str(prev_row[key])
-                curr_key = str(curr_row[key])
+                prev_key = prev_row[key]
+                curr_key = curr_row[key]
                 if prev_key == curr_key:
                     # Check for changes
                     changed_fields = {
@@ -192,46 +190,40 @@ def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignor
                 curr_row = next(reader2, None)
     return result
 
-
 def human_text(result, key=None, current=None, extras=None):
+    """
+    Render the diff result as a human-readable string.
+    """
     title = []
     summary = []
-    show_headers = sum(1 for key in result if result[key]) > 1
+    show_headers = sum(1 for k in result if result[k]) > 1
     if result["columns_added"]:
-        fragment = "{} {} added".format(
-            len(result["columns_added"]),
-            "column" if len(result["columns_added"]) == 1 else "columns",
-        )
+        fragment = f"{len(result['columns_added'])} {'column' if len(result['columns_added']) == 1 else 'columns'} added"
         title.append(fragment)
         summary.extend(
             [fragment, ""]
-            + ["  {}".format(c) for c in sorted(result["columns_added"])]
+            + [f"  {c}" for c in sorted(result["columns_added"])]
             + [""]
         )
     if result["columns_removed"]:
-        fragment = "{} {} removed".format(
-            len(result["columns_removed"]),
-            "column" if len(result["columns_removed"]) == 1 else "columns",
-        )
+        fragment = f"{len(result['columns_removed'])} {'column' if len(result['columns_removed']) == 1 else 'columns'} removed"
         title.append(fragment)
         summary.extend(
             [fragment, ""]
-            + ["  {}".format(c) for c in sorted(result["columns_removed"])]
+            + [f"  {c}" for c in sorted(result["columns_removed"])]
             + [""]
         )
     if result["changed"]:
-        fragment = "{} rows changed".format(len(result["changed"]))
+        fragment = f"{len(result['changed'])} rows changed"
         title.append(fragment)
         if show_headers:
             summary.append(fragment + "\n")
         change_blocks = []
         for details in result["changed"]:
             block = []
-            block.append("  {}: {}".format(key, details["key"]))
+            block.append(f"  {key}: {details['key']}")
             for field, (prev_value, current_value) in details["changes"].items():
-                block.append(
-                    '    {}: "{}" => "{}"'.format(field, prev_value, current_value)
-                )
+                block.append(f'    {field}: "{prev_value}" => "{current_value}"')
             if extras:
                 current_item = current[details["key"]]
                 block.append(human_extras(current_item, extras))
@@ -241,12 +233,12 @@ def human_text(result, key=None, current=None, extras=None):
                 block = []
                 block.append("    Unchanged:")
                 for field, value in details["unchanged"].items():
-                    block.append('      {}: "{}"'.format(field, value))
+                    block.append(f'      {field}: "{value}"')
                 block.append("")
                 change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
     if result["added"]:
-        fragment = "{} rows added".format(len(result["added"]))
+        fragment = f"{len(result['added'])} rows added"
         title.append(fragment)
         if show_headers:
             summary.append(fragment + "\n")
@@ -259,7 +251,7 @@ def human_text(result, key=None, current=None, extras=None):
         summary.append("\n\n".join(rows))
         summary.append("")
     if result["removed"]:
-        fragment = "{} rows removed".format(len(result["removed"]))
+        fragment = f"{len(result['removed'])} rows removed"
         title.append(fragment)
         if show_headers:
             summary.append(fragment + "\n")
@@ -273,17 +265,17 @@ def human_text(result, key=None, current=None, extras=None):
         summary.append("")
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
-
 def human_row(row, prefix=""):
-    bits = []
-    for key, value in row.items():
-        bits.append("{}{}: {}".format(prefix, key, value))
-    return "\n".join(bits)
-
+    """
+    Render a row as a human-readable string.
+    """
+    return "\n".join(f"{prefix}{key}: {value}" for key, value in row.items())
 
 def human_extras(row, extras):
-    bits = []
-    bits.append("  extras:")
+    """
+    Render extra fields for a row.
+    """
+    bits = ["  extras:"]
     for key, fmt in extras:
-        bits.append("    {}: {}".format(key, fmt.format(**row)))
+        bits.append(f"    {key}: {fmt.format(**row)}")
     return "\n".join(bits)
diff --git a/db_diff/cli.py b/db_diff/cli.py
index ba3847f..9733232 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -3,8 +3,47 @@
 import time as time_module
 import os
 import datetime
+import csv
 from . import load_csv, load_json, compare, human_text, streaming_compare_csv
 
+DIALECTS = {
+    "csv": "excel",
+    "tsv": "excel-tab",
+}
+
+def sniff_key_column(filename, input_format, encoding="utf-8"):
+    """
+    Detect the key column from the first row of a CSV/TSV/JSON file.
+    """
+    try:
+        if input_format == "json":
+            with open(filename, encoding=encoding) as f:
+                data = std_json.load(f)
+                if isinstance(data, list) and data:
+                    return list(data[0].keys())[0]
+        else:
+            with open(filename, newline="", encoding=encoding) as f:
+                reader = csv.reader(f, dialect=DIALECTS.get(input_format))
+                headings = next(reader)
+                return headings[0]
+    except Exception as e:
+        raise click.ClickException(f"Could not sniff key column: {e}")
+    return None
+
+def load(filename, key, input_format, encoding="utf-8"):
+    """
+    Load a CSV/TSV/JSON file into a dictionary keyed by the given column.
+    """
+    try:
+        if input_format == "json":
+            with open(filename, encoding=encoding) as f:
+                return load_json(f, key=key)
+        else:
+            with open(filename, newline="", encoding=encoding) as f:
+                return load_csv(f, key=key, dialect=DIALECTS.get(input_format))
+    except Exception as e:
+        raise click.ClickException(f"Failed to load '{filename}': {e}")
+
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
     help="Compare the differences between two CSV or JSON files to find differences.",
@@ -86,20 +125,48 @@
     is_flag=True,
     help="Use streaming mode for very large CSV/TSV files (requires files to be sorted by key).",
 )
+@click.option(
+    "--list-fields",
+    is_flag=True,
+    help="List available fields/columns in the input files and exit.",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,
     show_unchanged, encoding, show_time, output, output_file, output_path,
-    fields, ignorefields, streaming
+    fields, ignorefields, streaming, list_fields
 ):
-    """Compare the differences between two CSV or JSON files."""
-    dialect = {
-        "csv": "excel",
-        "tsv": "excel-tab",
-    }
-
+    """
+    Compare the differences between two CSV or JSON files.
+    """
     click.echo(f"Starting diff: '{previous}' vs '{current}'...")
 
+    # Detect file format
+    detected_format = input_format or previous.split(".")[-1].lower()
+    if detected_format not in ("csv", "tsv", "json"):
+        raise click.ClickException(f"Unsupported file format: {detected_format}")
+
+    # List fields and exit
+    if list_fields:
+        try:
+            if detected_format == "json":
+                with open(previous, encoding=encoding) as f:
+                    data = std_json.load(f)
+                    if isinstance(data, list) and data:
+                        click.echo("Available fields:")
+                        for field in data[0].keys():
+                            click.echo(f"  {field}")
+            else:
+                with open(previous, newline="", encoding=encoding) as f:
+                    reader = csv.reader(f, dialect=DIALECTS.get(detected_format))
+                    headings = next(reader)
+                    click.echo("Available fields:")
+                    for field in headings:
+                        click.echo(f"  {field}")
+        except Exception as e:
+            raise click.ClickException(f"Could not list fields: {e}")
+        return
+
     # Set default output filename if needed
     if output == "jsonfile":
         if not output_file:
@@ -110,40 +177,9 @@ def cli(
                 raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
             output_file = os.path.join(output_path, os.path.basename(output_file))
 
-    def sniff_key_column(filename, input_format):
-        # Only for CSV/TSV/JSON
-        if input_format == "json":
-            with open(filename, encoding=encoding) as f:
-                import json
-                data = json.load(f)
-                if isinstance(data, list) and data:
-                    return list(data[0].keys())[0]
-        else:
-            with open(filename, newline="", encoding=encoding) as f:
-                import csv
-                reader = csv.reader(f, dialect=dialect.get(input_format))
-                headings = next(reader)
-                return headings[0]
-        return None
-
-    def load(filename, key):
-        try:
-            if input_format == "json":
-                with open(filename, encoding=encoding) as f:
-                    return load_json(f, key=key)
-            else:
-                with open(filename, newline="", encoding=encoding) as f:
-                    return load_csv(f, key=key, dialect=dialect.get(input_format))
-        except Exception as e:
-            raise click.ClickException(f"Failed to load '{filename}': {e}")
-
-    if show_time:
-        start = time_module.time()
-
     # Detect key column before loading data
     if key is None:
-        detected_format = input_format or previous.split(".")[-1].lower()
-        key = sniff_key_column(previous, detected_format)
+        key = sniff_key_column(previous, detected_format, encoding)
         if not key:
             raise click.ClickException("Could not determine key column from the previous file.")
 
@@ -155,11 +191,12 @@ def load(filename, key):
     fields_set = set(f.strip() for f in fields.split(",")) if fields else None
     ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
 
+    if show_time:
+        start = time_module.time()
+
     # --- Streaming logic ---
-    if streaming and (input_format in ("csv", "tsv", None)):
-        # Default to csv if not specified
-        fmt = input_format or previous.split(".")[-1].lower()
-        if fmt not in ("csv", "tsv"):
+    if (streaming or (detected_format in ("csv", "tsv") and key)) and not fields:
+        if detected_format not in ("csv", "tsv"):
             raise click.ClickException("--streaming only works with CSV/TSV files.")
         diff = streaming_compare_csv(
             previous,
@@ -168,13 +205,13 @@ def load(filename, key):
             compare_columns=fields_set,
             ignorefields=ignorefields_set,
             encoding=encoding,
-            dialect=dialect.get(fmt, "excel"),
+            dialect=DIALECTS.get(detected_format, "excel"),
         )
         # For human_text, we need current_data for extras (if used)
         current_data = None
     else:
-        previous_data = load(previous, key)
-        current_data = load(current, key)
+        previous_data = load(previous, key, detected_format, encoding)
+        current_data = load(current, key, detected_format, encoding)
         diff = compare(
             previous_data,
             current_data,

From 17af5cf58e63abcd4b9719afc2c57d3a8edc3062 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 16:48:58 -0500
Subject: [PATCH 072/129] made listfields option naming consistent

---
 db_diff/cli.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 9733232..1450975 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -126,7 +126,7 @@ def load(filename, key, input_format, encoding="utf-8"):
     help="Use streaming mode for very large CSV/TSV files (requires files to be sorted by key).",
 )
 @click.option(
-    "--list-fields",
+    "--listfields",
     is_flag=True,
     help="List available fields/columns in the input files and exit.",
 )
@@ -134,7 +134,7 @@ def load(filename, key, input_format, encoding="utf-8"):
 def cli(
     previous, current, key, input_format,
     show_unchanged, encoding, show_time, output, output_file, output_path,
-    fields, ignorefields, streaming, list_fields
+    fields, ignorefields, streaming, listfields
 ):
     """
     Compare the differences between two CSV or JSON files.
@@ -147,7 +147,7 @@ def cli(
         raise click.ClickException(f"Unsupported file format: {detected_format}")
 
     # List fields and exit
-    if list_fields:
+    if listfields:
         try:
             if detected_format == "json":
                 with open(previous, encoding=encoding) as f:

From e4d5e27bc83573a6792e28ce427b4e0b7842d73e Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 16:51:47 -0500
Subject: [PATCH 073/129] updated setup file

---
 setup.py | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/setup.py b/setup.py
index 64edec4..c426d9a 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,6 @@
 
 VERSION = "3.3"
 
-
 def get_long_description():
     with io.open(
         os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"),
@@ -12,22 +11,27 @@ def get_long_description():
     ) as fp:
         return fp.read()
 
-
 setup(
     name="db-diff",
-    description="Python CLI tool and library for comparing CSV database dumps and finding differences.",
+    description="Python CLI tool and library for comparing CSV/TSV/JSON database dumps and finding differences.",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
     author="Thomas Coyle",
     version=VERSION,
     license="Apache License, Version 2.0",
     packages=find_packages(),
-    install_requires=["click", "dictdiffer"],
-    entry_points="""
-        [console_scripts]
-        db-diff=db_diff.cli:cli
-    """,
+    install_requires=[
+        "click>=7.0",
+        "dictdiffer>=0.8.0",
+        "xxhash>=3.0.0"
+    ],
+    entry_points={
+        "console_scripts": [
+            "db-diff=db_diff.cli:cli"
+        ]
+    },
     url="https://github.com/datsom1/db-diff",
+    python_requires=">=3.6",
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",
@@ -36,5 +40,15 @@ def get_long_description():
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Operating System :: OS Independent",
+        "Topic :: Utilities",
+        "Topic :: Software Development :: Libraries :: Python Modules",
     ],
+    include_package_data=True,
+    zip_safe=False,
 )

From 4a4a7de4109dc5bc694cd94076a128f94f73f0bd Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 9 Jun 2025 17:01:29 -0500
Subject: [PATCH 074/129] Updated readme

---
 README.md | 403 ++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 288 insertions(+), 115 deletions(-)

diff --git a/README.md b/README.md
index e34b480..89eaed1 100644
--- a/README.md
+++ b/README.md
@@ -3,156 +3,329 @@
 [![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
 
-Python CLI tool and library for comparing CSV database dumps and finding differences.
+> **db-diff** is a Python CLI tool and library for comparing CSV, TSV, and JSON database dumps, highlighting differences in a human-readable or machine-readable format.
+
+---
+
+## Table of Contents
+
+- [Features](#features)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+- [Usage](#usage)
+  - [Command Line](#command-line)
+  - [Python Library](#python-library)
+- [Options](#options)
+- [Examples](#examples)
+- [Performance](#performance)
+- [Development](#development)
+- [License](#license)
+
+---
+
+## Features
+
+- Compare CSV, TSV, and JSON files for differences
+- Human-readable and machine-readable (JSON) output
+- Detects added, removed, and changed rows and columns
+- Supports custom key columns for row identity
+- Field inclusion/exclusion for focused diffs
+- Streaming mode for large files (memory efficient)
+- Output to terminal or JSON file
+- Automatic delimiter and encoding detection
+- Python 3.6+ compatible
+
+---
 
 ## Installation
 
-    pip install git+https://github.com/datsom1/db-diff.git
+Install the latest version directly from GitHub:
+
+```sh
+pip install git+https://github.com/datsom1/db-diff.git
+```
+
+To upgrade to the latest version:
+
+```sh
+pip install --upgrade --force-reinstall git+https://github.com/datsom1/db-diff.git
+```
+
+---
+
+## Quick Start
+
+Suppose you have two CSV files:
+
+**one.csv**
+```
+Id,name,age
+1,Cleo,4
+2,Pancakes,2
+```
+
+**two.csv**
+```
+Id,name,age
+1,Cleo,5
+3,Bailey,1
+```
 
-To install the unofficial latest version (you probably don't need to do this):
+Compare them using:
 
-    pip install --upgrade --force-reinstall git+https://github.com/datsom1/db-diff.git
+```sh
+db-diff one.csv two.csv --key=Id
+```
+
+Sample output:
+
+```
+1 rows changed, 1 rows added, 1 rows removed
+
+1 rows changed
+
+  Id: 1
+    age: "4" => "5"
+
+1 rows added
+
+  Id: 3
+  name: Bailey
+  age: 1
+
+1 rows removed
+
+  Id: 2
+  name: Pancakes
+  age: 2
+```
+
+---
 
 ## Usage
 
-Consider two CSV files:
+### Command Line
+
+`db-diff` is a flexible CLI tool for comparing two data files (CSV, TSV, or JSON). It detects added, removed, and changed rows and columns, and can output results in a human-readable or machine-readable format.
+
+**Basic usage:**
+
+```sh
+db-diff [OPTIONS] PREVIOUS CURRENT
+```
+
+- `PREVIOUS` and `CURRENT` are the file paths to the two files you want to compare.
+- The tool auto-detects file format by extension, or you can specify with `--format`.
+
+**Key features:**
+
+- **Custom Key Column:** Use `--key` to specify which column uniquely identifies rows.
+- **Output Formats:** Choose between human-readable (`readable`), JSON (`json`), or save JSON to a file (`jsonfile`).
+- **Field Selection:** Use `--fields` to compare only specific columns, or `--ignorefields` to exclude columns.
+- **Streaming Mode:** For very large files, use `--streaming` (CSV/TSV only, files must be sorted by key).
+- **Encoding:** Specify file encoding with `--encoding`.
+- **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
+- **List Fields:** Use `--listfields` to print available columns and exit.
+- **Timing:** Use `--time` to display how long the diff operation took.
+
+**See all options:**
+
+```sh
+db-diff --help
+```
+
+---
+
+### Python Library
+
+You can use `db-diff` as a Python library for advanced or automated workflows. The library provides functions to load data, compare datasets, and render results.
+
+#### Loading Data
+
+```python
+from db_diff import load_csv, load_json
+
+# Load CSV file, using a specific column as the key
+with open("one.csv", encoding="utf-8") as f:
+    prev = load_csv(f, key="Id")
+
+# Load JSON file, using a specific key
+with open("two.json", encoding="utf-8") as f:
+    curr = load_json(f, key="Id")
+```
+
+#### Comparing Data
+
+```python
+from db_diff import compare
+
+# Compare two datasets (dictionaries keyed by your chosen column)
+diff = compare(prev, curr, show_unchanged=False)
+```
+
+- `show_unchanged`: If True, includes unchanged fields for changed rows.
+- `fields`: Pass a set of field names to only compare those fields.
+- `ignorefields`: Pass a set of field names to ignore during comparison.
+
+#### Streaming Comparison (for large CSV/TSV files)
+
+```python
+from db_diff import streaming_compare_csv
+
+diff = streaming_compare_csv(
+    "one.csv",
+    "two.csv",
+    key="Id",
+    compare_columns={"Id", "name", "age"},
+    encoding="utf-8",
+    dialect="excel"
+)
+```
+
+#### Diff Result Structure
+
+The result of `compare` or `streaming_compare_csv` is a dictionary:
+
+```python
+{
+    "added": [ ... ],            # List of added rows (dicts)
+    "removed": [ ... ],          # List of removed rows (dicts)
+    "changed": [                 # List of changed rows
+        {
+            "key": "row_id",
+            "changes": {
+                "field1": ["old", "new"],
+                ...
+            },
+            "unchanged": { ... } # (optional) if show_unchanged=True
+        },
+        ...
+    ],
+    "columns_added": [ ... ],    # List of columns added
+    "columns_removed": [ ... ]   # List of columns removed
+}
+```
+
+#### Rendering Human-Readable Output
+
+```python
+from db_diff import human_text
+
+print(human_text(diff, key="Id", current=curr))
+```
+
+#### Example: Full Workflow
+
+```python
+from db_diff import load_csv, compare, human_text
+
+with open("one.csv") as f1, open("two.csv") as f2:
+    prev = load_csv(f1, key="Id")
+    curr = load_csv(f2, key="Id")
+    diff = compare(prev, curr, show_unchanged=True)
+    print(human_text(diff, key="Id", current=curr))
+```
+
+---
+
+## Options
+
+See all available options with:
+
+```sh
+db-diff --help
+```
 
-`one.csv`
+A summary of key options:
 
-    Id,name,age
-    1,Cleo,4
-    2,Pancakes,2
+| Option                | Description                                                                                  |
+|-----------------------|----------------------------------------------------------------------------------------------|
+| `--key TEXT`          | Column to use as a unique ID for each row (default: first column header)                     |
+| `--output TEXT`       | Output format: `readable`, `json`, or `jsonfile` (default: readable)                         |
+| `--outfilename FILE`  | File to write JSON output to (used with `--output=jsonfile`)                                 |
+| `--outfilepath DIR`   | Directory to save the output file (used with `--output=jsonfile`)                            |
+| `--fields TEXT`       | Comma-separated list of fields to compare (all others ignored)                               |
+| `--ignorefields TEXT` | Comma-separated list of fields to ignore during comparison                                   |
+| `--showunchanged`     | Show all fields for changed records, not just changed fields                                 |
+| `--time`              | Measure and display elapsed time for the diff operation                                      |
+| `--format TEXT`       | Explicitly specify input format: `csv`, `tsv`, or `json` (default: auto-detect)              |
+| `--encoding TEXT`     | Input file encoding (default: utf-8)                                                         |
+| `--streaming`         | Use streaming mode for very large CSV/TSV files (requires files to be sorted by key)         |
+| `--listfields`        | List available fields/columns in the input files and exit                                    |
+| `--version`           | Show the version and exit                                                                    |
+| `-h, --help`          | Show help message and exit                                                                   |
 
-`two.csv`
+---
 
-    Id,name,age
-    1,Cleo,5
-    3,Bailey,1
+## Examples
 
-`db-diff` can show a human-readable summary of differences between the files:
+**Show unchanged fields for changed rows:**
 
-    $ db-diff one.csv two.csv --key=Id
-    1 rows changed, 1 rows added, 1 rows removed
+```sh
+db-diff one.csv two.csv --key=Id --showunchanged
+```
 
-    1 rows changed
+**Output as JSON:**
 
-      Rows 1
-        age: "4" => "5"
+```sh
+db-diff one.csv two.csv --key=Id --output=json
+```
 
-    1 rows added
+**Save JSON output to a file:**
 
-      Id: 3
-      name: Bailey
-      age: 1
+```sh
+db-diff one.csv two.csv --key=Id --output=jsonfile --outfilename=diffs.json
+```
 
-    1 rows removed
+**Compare only specific fields:**
 
-      Id: 2
-      name: Pancakes
-      age: 2
+```sh
+db-diff one.csv two.csv --key=Id --fields=Id,name
+```
 
-The `--key=Id` option means that the `Id` column should be treated as the unique key, to identify which records have changed.
+**Ignore specific fields:**
 
-The tool will automatically detect if your files are comma- or tab-separated. You can over-ride this automatic detection and force the tool to use a specific format using `--format=tsv` or `--format=csv`.
+```sh
+db-diff one.csv two.csv --key=Id --ignorefields=LastModifiedDate
+```
 
-You can also feed it JSON files, provided they are a JSON array of objects where each object has the same keys. Use `--format=json` if your input files are JSON.
+**Streaming mode for large files:**
 
-Use `--show-unchanged` to include full details of the unchanged values for rows with at least one change in the diff output:
+```sh
+db-diff large1.csv large2.csv --key=Id --streaming
+```
 
-    % db-diff one.csv two.csv --key=Id --showunchanged
-    1 rows changed
+---
 
-      Id: 1
-        age: "4" => "5"
+## Performance
 
-        Unchanged:
-          name: "Cleo"
+- For very large files, use `--streaming` mode (CSV/TSV only, files must be sorted by key).
+- Supports efficient memory usage and fast comparison for millions of rows.
 
-### JSON output
+---
 
-You can use the `--output=json` option to get a machine-readable difference:
+## Development
 
-    $ db-diff one.csv two.csv --key=Id --output=json
-    {
-        "added": [
-            {
-                "id": "3",
-                "name": "Bailey",
-                "age": "1"
-            }
-        ],
-        "removed": [
-            {
-                "id": "2",
-                "name": "Pancakes",
-                "age": "2"
-            }
-        ],
-        "changed": [
-            {
-                "key": "1",
-                "changes": {
-                    "age": [
-                        "4",
-                        "5"
-                    ]
-                }
-            }
-        ],
-        "columns_added": [],
-        "columns_removed": []
-    }
+Clone the repository:
 
-### JSON file output
+```sh
+git clone https://github.com/datsom1/db-diff.git
+cd db-diff
+```
 
-You can use the `--output=jsonfile` and `--outputfile=` option to automatically save a .json file of the output:
+Install dependencies:
 
-    $ db-diff one.csv two.csv --key=Id --output=jsonfile --outputfile=diffs.json
+```sh
+pip install -e .
+```
 
-### Measure time
+---
 
-You can use the `--time` option to meaure the time it takes:
+## License
 
-    $ db-diff one.csv two.csv --key=Id --time
-    .
-    .
-    .
-    Elapsed time: 0.016 seconds
+This project is licensed under the [Apache License 2.0](LICENSE).
 
-## As a Python library
-
-You can also import the Python library into your own code like so:
+---
 
-    from csv_diff import load_csv, compare
-    diff = compare(
-        load_csv(open("one.csv"), key="Id"),
-        load_csv(open("two.csv"), key="Id")
-    )
-
-`diff` will now contain the same data structure as the output in the `--output=json` example above.
-
-If the columns in the CSV have changed, those added or removed columns will be ignored when calculating changes made to specific rows.
-
-## Full CLI Helptext
-
-    $ db-diff --help
-    Usage: db-diff [OPTIONS] PREVIOUS CURRENT
-    
-      Compare the differences between two CSV or JSON files to find differences.
-    
-    Options:
-      --key TEXT               Column to use as a unique ID for each row (ex: --key=Id). [default: first column header]
-      --output TEXT            Output format. Available: (readable|json|jsonfile). [default: readable]
-      --outfilename FILE       File to write JSON output to (only used with --output=jsonfile). [default: YYYY-MM-DD_HH-MM-SS_diffs.json]
-      --outfilepath DIRECTORY  Directory to save the output file (only used with --output=jsonfile). [default: current directory]
-      --fields TEXT            Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age__c,Email__c).
-      --ignorefields TEXT      Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp,Days_Since_Last_Contact__c).
-      --showunchanged          If a record is changed, show ALL fields, not just the changed fields.
-      --time                   Measure and display elapsed time for the diff operation.
-      --format TEXT            Explicitly specify input format. Available (csv|tsv|json). [default: auto-detect based on file extension]
-      --encoding TEXT          Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).  [default: utf-8]
-      --streaming              Use streaming mode for very large CSV/TSV files (requires files to be sorted by key).
-      --version                Show the version and exit.
-      -h, --help               Show this message and exit.
-    
-      Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outfilename=diff.json
+**Author:** Thomas Coyle  
+**Repository:** https://github.com/datsom1/db-diff

From 9e3d9b5f0ed8e0e6807b5aa03e579baa0519c1e4 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 10 Jun 2025 10:22:01 -0500
Subject: [PATCH 075/129] update

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 89eaed1..f3a5d5b 100644
--- a/README.md
+++ b/README.md
@@ -27,10 +27,11 @@
 
 - Compare CSV, TSV, and JSON files for differences
 - Human-readable and machine-readable (JSON) output
+- JSON output to file with filename and path support
 - Detects added, removed, and changed rows and columns
 - Supports custom key columns for row identity
 - Field inclusion/exclusion for focused diffs
-- Streaming mode for large files (memory efficient)
+- Streaming mode for very large files (memory efficient)
 - Output to terminal or JSON file
 - Automatic delimiter and encoding detection
 - Python 3.6+ compatible

From aead51205585cced351c8bdb8b9c06a94bd65217 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 10 Jun 2025 10:22:36 -0500
Subject: [PATCH 076/129] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f3a5d5b..914f947 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@
 - Supports custom key columns for row identity
 - Field inclusion/exclusion for focused diffs
 - Streaming mode for very large files (memory efficient)
-- Output to terminal or JSON file
+- Can output to terminal or JSON file
 - Automatic delimiter and encoding detection
 - Python 3.6+ compatible
 

From 15ac33f402cf827d0ae1ff67f09f9e02feafc96e Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 10 Jun 2025 10:26:43 -0500
Subject: [PATCH 077/129] update

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 914f947..a2b836c 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 [![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
 
-> **db-diff** is a Python CLI tool and library for comparing CSV, TSV, and JSON database dumps, highlighting differences in a human-readable or machine-readable format.
+> **db-diff** is the most advanced and user-friendly Python CLI tool and library for comparing CSV, TSV, and JSON database dumps. Designed specifically for database workflows, it delivers precise, human-readable or machine-readable diffs, supports custom key columns, handles massive files efficiently with streaming mode, and offers flexible field selection. db-diff makes database comparison fast, accurate, and effortless—making it the best choice for database professionals and data engineers.
 
 ---
 

From 325405782a3df11c83ff117e375e64f1499fb796 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 10 Jun 2025 14:14:59 -0500
Subject: [PATCH 078/129] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2b836c..7964ae5 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 [![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
 
-> **db-diff** is the most advanced and user-friendly Python CLI tool and library for comparing CSV, TSV, and JSON database dumps. Designed specifically for database workflows, it delivers precise, human-readable or machine-readable diffs, supports custom key columns, handles massive files efficiently with streaming mode, and offers flexible field selection. db-diff makes database comparison fast, accurate, and effortless—making it the best choice for database professionals and data engineers.
+> **db-diff** is the most advanced and user-friendly Python CLI tool and library for comparing CSV, TSV, and JSON database dumps. Designed specifically for database workflows, it delivers precise, human-readable or machine-readable diffs, supports custom key columns, handles massive files efficiently with streaming mode, and offers flexible field selection.
 
 ---
 

From cc9ee685a3341eae0ab8a3ade6c6e90ac9b5cbe9 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Thu, 12 Jun 2025 15:30:14 -0500
Subject: [PATCH 079/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c426d9a..5b48526 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "3.3"
+VERSION = "3.6"
 
 def get_long_description():
     with io.open(

From a0236a599cb994a3574c292a641feb4b2aa7f6cd Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Wed, 25 Jun 2025 15:01:28 -0500
Subject: [PATCH 080/129] made it so that the help text will show if no args
 are input

---
 db_diff/cli.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 1450975..920f22d 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -48,6 +48,7 @@ def load(filename, key, input_format, encoding="utf-8"):
     context_settings={"help_option_names": ["-h", "--help"]},
     help="Compare the differences between two CSV or JSON files to find differences.",
     epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outfilename=diff.json",
+    no_args_is_help=True,
 )
 @click.argument(
     "previous",

From 6b5840cf2cf18e368cb347593b665fa3242e5e11 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Wed, 25 Jun 2025 15:06:38 -0500
Subject: [PATCH 081/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5b48526..a2bae95 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "3.6"
+VERSION = "3.7"
 
 def get_long_description():
     with io.open(

From 07b662729435db59719791a4d64951b654328d5b Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 11:41:44 -0500
Subject: [PATCH 082/129] updated how --outfilename works

---
 db_diff/cli.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 920f22d..dde1903 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -170,9 +170,14 @@ def cli(
 
     # Set default output filename if needed
     if output == "jsonfile":
+        timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
         if not output_file:
-            timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
             output_file = f"{timestamp}_diffs.json"
+        else:
+            # Append the user-specified filename to the timestamp
+            base_name = os.path.splitext(output_file)[0]  # Remove extension if present
+            output_file = f"{timestamp}_diffs_{base_name}.json"
+        
         if output_path:
             if not os.path.isdir(output_path):
                 raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")

From 796034a966502a0d2335b89526f7009d187575ca Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 11:58:47 -0500
Subject: [PATCH 083/129] updated base filename

---
 db_diff/cli.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index dde1903..24cfb3f 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -171,8 +171,14 @@ def cli(
     # Set default output filename if needed
     if output == "jsonfile":
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        
+        # Extract base filenames without extensions for the comparison files
+        prev_basename = os.path.splitext(os.path.basename(previous))[0]
+        curr_basename = os.path.splitext(os.path.basename(current))[0]
+        default_name = f"{timestamp}_diffs_{prev_basename}__VS__{curr_basename}.json"
+        
         if not output_file:
-            output_file = f"{timestamp}_diffs.json"
+            output_file = default_name
         else:
             # Append the user-specified filename to the timestamp
             base_name = os.path.splitext(output_file)[0]  # Remove extension if present

From 57dc077b35e5dfaa5ee457c91cb5cdb6a824d275 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 13:30:59 -0500
Subject: [PATCH 084/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a2bae95..5bf3248 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "3.7"
+VERSION = "3.8"
 
 def get_long_description():
     with io.open(

From 8bacb8848c22692fbc9e2098f23cdd1c248cf90a Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 13:32:28 -0500
Subject: [PATCH 085/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5bf3248..66682f6 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,7 @@ def get_long_description():
     url="https://github.com/datsom1/db-diff",
     python_requires=">=3.6",
     classifiers=[
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
         "Intended Audience :: Science/Research",
         "Intended Audience :: End Users/Desktop",

From fc521a96aaf88c3f1384b16009e04dd8cdcd87a1 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 13:34:02 -0500
Subject: [PATCH 086/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 66682f6..bc3b8b3 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "3.8"
+VERSION = "4.0"
 
 def get_long_description():
     with io.open(

From 000f8759648460d8d94dd04d47f52c47214b9baf Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 13:36:17 -0500
Subject: [PATCH 087/129] updated helptext

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 24cfb3f..a30138e 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -75,7 +75,7 @@ def load(filename, key, input_format, encoding="utf-8"):
     "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
-    help="File to write JSON output to (only used with --output=jsonfile). [default: YYYY-MM-DD_HH-MM-SS_diffs.json]",
+    help="File to write JSON output to (only used with --output=jsonfile). [default: YYYY-MM-DD_HH-MM-SS_diffs_file1name__VS__file2name.json]",
 )
 @click.option(
     "--outfilepath",

From 2516a3e671143dcbc0d365c6cdbf5c2e36c84128 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 13:43:16 -0500
Subject: [PATCH 088/129] better terminal width detection

---
 db_diff/cli.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index a30138e..bd72ad9 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -4,8 +4,12 @@
 import os
 import datetime
 import csv
+import shutil  # Add this import
 from . import load_csv, load_json, compare, human_text, streaming_compare_csv
 
+# Get terminal width for better help formatting
+TERMINAL_WIDTH = shutil.get_terminal_size().columns
+
 DIALECTS = {
     "csv": "excel",
     "tsv": "excel-tab",
@@ -45,7 +49,10 @@ def load(filename, key, input_format, encoding="utf-8"):
         raise click.ClickException(f"Failed to load '{filename}': {e}")
 
 @click.command(
-    context_settings={"help_option_names": ["-h", "--help"]},
+    context_settings={
+        "help_option_names": ["-h", "--help"],
+        "max_content_width": TERMINAL_WIDTH  # Set the max width to terminal width
+    },
     help="Compare the differences between two CSV or JSON files to find differences.",
     epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outfilename=diff.json",
     no_args_is_help=True,

From a6a0edb3313cd532d46b18f0e9c81e8e8d9fb50b Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 13:44:30 -0500
Subject: [PATCH 089/129] updated helptext

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index bd72ad9..fd3c31e 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -101,7 +101,7 @@ def load(filename, key, input_format, encoding="utf-8"):
     "--ignorefields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp,Days_Since_Last_Contact__c).",
+    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp).",
 )
 @click.option(
     "--showunchanged",

From 17d26cf7fe9b4be8995eaa2899ab4e621572cc13 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 13:47:44 -0500
Subject: [PATCH 090/129] Removed the --time option and made program always
 show time

---
 db_diff/cli.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index fd3c31e..db20f15 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -109,12 +109,6 @@ def load(filename, key, input_format, encoding="utf-8"):
     is_flag=True,
     help="If a record is changed, show ALL fields, not just the changed fields.",
 )
-@click.option(
-    "--time",
-    "show_time",
-    is_flag=True,
-    help="Measure and display elapsed time for the diff operation.",
-)
 @click.option(
     "--format",
     "input_format",
@@ -141,7 +135,7 @@ def load(filename, key, input_format, encoding="utf-8"):
 @click.version_option()
 def cli(
     previous, current, key, input_format,
-    show_unchanged, encoding, show_time, output, output_file, output_path,
+    show_unchanged, encoding, output, output_file, output_path,
     fields, ignorefields, streaming, listfields
 ):
     """
@@ -210,8 +204,8 @@ def cli(
     fields_set = set(f.strip() for f in fields.split(",")) if fields else None
     ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
 
-    if show_time:
-        start = time_module.time()
+    # Always measure time (removed the --time option)
+    start = time_module.time()
 
     # --- Streaming logic ---
     if (streaming or (detected_format in ("csv", "tsv") and key)) and not fields:
@@ -252,6 +246,6 @@ def cli(
         )
         click.echo("Finished. Output printed in readable format to terminal.")
 
-    if show_time:
-        end = time_module.time()
-        print(f"\nElapsed time: {end - start:.3f} seconds")
+    # Always display the elapsed time
+    end = time_module.time()
+    print(f"\nElapsed time: {end - start:.3f} seconds")

From cb8fe7a4cedde79508a41d9c6c66fa469e378b21 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:09:08 -0500
Subject: [PATCH 091/129] code refactoring

---
 db_diff/__init__.py | 252 ++++++++++++++++++++++++++++++++---------
 db_diff/cli.py      | 270 +++++++++++++++++++++++++++++++++-----------
 setup.py            |   6 +-
 3 files changed, 403 insertions(+), 125 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 72f89d7..b497690 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -1,11 +1,33 @@
+"""
+DB-Diff - A tool for comparing database dumps in CSV or JSON format.
+
+This module provides functions for loading and comparing data from CSV and JSON files,
+generating diffs, and displaying results in human-readable format.
+"""
+
 import csv
 import json
 import xxhash
 from dictdiffer import diff
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+logger = logging.getLogger(__name__)
+
+__version__ = "1.0.0"
 
 def load_csv(fp, key=None, dialect=None):
     """
     Load a CSV file into a dictionary keyed by the given column or hash.
+    
+    Args:
+        fp: File-like object containing CSV data
+        key: Column name to use as the dictionary key (optional)
+        dialect: CSV dialect to use (optional)
+        
+    Returns:
+        Dictionary of rows keyed by the specified column or hash
     """
     if dialect is None and fp.seekable():
         # Peek at first 1MB to sniff the delimiter and other dialect details
@@ -14,6 +36,7 @@ def load_csv(fp, key=None, dialect=None):
         try:
             dialect = csv.Sniffer().sniff(peek, delimiters=",\t;")
         except csv.Error:
+            logger.debug("Could not sniff CSV dialect, falling back to default")
             # Oh well, we tried. Fallback to the default.
             pass
     reader = csv.reader(fp, dialect=(dialect or "excel"))
@@ -28,34 +51,76 @@ def load_csv(fp, key=None, dialect=None):
 def load_json(fp, key=None):
     """
     Load a JSON array of objects into a dictionary keyed by the given column or hash.
+    
+    Args:
+        fp: File-like object containing JSON data
+        key: Key to use for dictionary indexing (optional)
+        
+    Returns:
+        Dictionary of rows keyed by the specified column or hash
+        
+    Raises:
+        ValueError: If the JSON file doesn't contain a list of objects
     """
     raw_list = json.load(fp)
     if not isinstance(raw_list, list):
         raise ValueError("JSON file must contain a list of objects.")
+    if not raw_list:
+        logger.warning("JSON file contains an empty list")
+        return {}
+        
     common_keys = set()
     for item in raw_list:
         common_keys.update(item.keys())
+        
+    if key and key not in common_keys:
+        logger.warning(f"Key '{key}' not found in all JSON objects, some items may be skipped")
+        
     if key:
-        keyfn = lambda r: r[key]
+        keyfn = lambda r: r.get(key, str(id(r)))
     else:
         keyfn = lambda r: xxhash.xxh64(json.dumps(r, sort_keys=True).encode("utf8")).hexdigest()
+        
     return {keyfn(r): _simplify_json_row(r, common_keys) for r in raw_list}
 
 def _simplify_json_row(r, common_keys):
     """
     Ensure all rows have the same keys and serialize nested structures.
+    
+    Args:
+        r: Row data (dictionary)
+        common_keys: Set of all keys found across all rows
+        
+    Returns:
+        Processed row with standardized format
     """
-    for key, value in r.items():
+    result = r.copy()  # Create a copy to avoid modifying the original
+    
+    # Serialize nested structures
+    for key, value in result.items():
         if isinstance(value, (dict, tuple, list)):
-            r[key] = json.dumps(value)
+            result[key] = json.dumps(value)
+            
+    # Ensure all rows have the same keys
     for key in common_keys:
-        if key not in r:
-            r[key] = None
-    return r
+        if key not in result:
+            result[key] = None
+            
+    return result
 
 def compare(previous, current, show_unchanged=False, fields=None, ignorefields=None):
     """
     Compare two dictionaries of rows and return a diff summary.
+    
+    Args:
+        previous: Dictionary of previous data rows
+        current: Dictionary of current data rows
+        show_unchanged: Whether to include unchanged fields in the output
+        fields: List of specific fields to compare (optional)
+        ignorefields: List of fields to ignore in comparison (optional)
+        
+    Returns:
+        Dictionary containing diff information
     """
     result = {
         "added": [],
@@ -64,6 +129,15 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
         "columns_added": [],
         "columns_removed": [],
     }
+    
+    if not previous or not current:
+        logger.warning("One of the datasets is empty, comparison may be incomplete")
+        if not previous:
+            result["added"] = list(current.values())
+        if not current:
+            result["removed"] = list(previous.values())
+        return result
+        
     # Have the columns changed?
     previous_columns = set(next(iter(previous.values())).keys())
     current_columns = set(next(iter(current.values())).keys())
@@ -87,6 +161,8 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
     added = [id for id in current if id not in previous]
     removed_or_added = set(removed) | set(added)
     potential_changes = [id for id in current if id not in removed_or_added]
+    
+    # Find changed rows by comparing values
     changed = [
         id for id in potential_changes
         if any(
@@ -94,6 +170,8 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
             for k in compare_columns
         )
     ]
+    
+    # Build result dictionary
     if added:
         result["added"] = [current[id] for id in added]
     if removed:
@@ -123,11 +201,24 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
                         if field not in changes["changes"] and field != "id" and field in compare_columns
                     }
                 result["changed"].append(changes)
+                
     return result
 
 def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignorefields=None, encoding='utf-8', dialect='excel'):
     """
     Efficiently compare two sorted CSV files by streaming, returning a diff dict.
+    
+    Args:
+        prev_path: Path to the previous CSV file
+        curr_path: Path to the current CSV file
+        key: Column name to use as key for comparison
+        compare_columns: List of columns to compare (optional)
+        ignorefields: List of columns to ignore (optional)
+        encoding: File encoding (default: utf-8)
+        dialect: CSV dialect (default: excel)
+        
+    Returns:
+        Dictionary containing diff information
     """
     result = {
         "added": [],
@@ -136,67 +227,95 @@ def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignor
         "columns_added": [],
         "columns_removed": [],
     }
-    with open(prev_path, newline='', encoding=encoding) as f1, open(curr_path, newline='', encoding=encoding) as f2:
-        reader1 = csv.DictReader(f1, dialect=dialect)
-        reader2 = csv.DictReader(f2, dialect=dialect)
-        prev_row = next(reader1, None)
-        curr_row = next(reader2, None)
+    
+    try:
+        with open(prev_path, newline='', encoding=encoding) as f1, open(curr_path, newline='', encoding=encoding) as f2:
+            reader1 = csv.DictReader(f1, dialect=dialect)
+            reader2 = csv.DictReader(f2, dialect=dialect)
+            
+            if not reader1.fieldnames or not reader2.fieldnames:
+                logger.error("One or both CSV files appear to be empty or malformed")
+                return result
+                
+            if key not in reader1.fieldnames:
+                raise ValueError(f"Key column '{key}' not found in previous file")
+            if key not in reader2.fieldnames:
+                raise ValueError(f"Key column '{key}' not found in current file")
+                
+            prev_row = next(reader1, None)
+            curr_row = next(reader2, None)
 
-        prev_columns = set(reader1.fieldnames)
-        curr_columns = set(reader2.fieldnames)
+            prev_columns = set(reader1.fieldnames)
+            curr_columns = set(reader2.fieldnames)
 
-        # Determine columns to compare
-        if compare_columns:
-            compare_columns = set(compare_columns)
-        elif ignorefields:
-            compare_columns = (prev_columns | curr_columns) - set(ignorefields)
-        else:
-            compare_columns = prev_columns | curr_columns
+            # Determine columns to compare
+            if compare_columns:
+                compare_columns = set(compare_columns)
+            elif ignorefields:
+                compare_columns = (prev_columns | curr_columns) - set(ignorefields)
+            else:
+                compare_columns = prev_columns | curr_columns
 
-        result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns]
-        result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns]
+            result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns]
+            result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns]
 
-        while prev_row or curr_row:
-            if prev_row and curr_row:
-                prev_key = prev_row[key]
-                curr_key = curr_row[key]
-                if prev_key == curr_key:
-                    # Check for changes
-                    changed_fields = {
-                        col: [prev_row.get(col), curr_row.get(col)]
-                        for col in compare_columns
-                        if prev_row.get(col) != curr_row.get(col)
-                    }
-                    if changed_fields:
-                        result["changed"].append({
-                            "key": prev_key,
-                            "changes": changed_fields
-                        })
-                    prev_row = next(reader1, None)
-                    curr_row = next(reader2, None)
-                elif prev_key < curr_key:
-                    # Row removed
+            while prev_row or curr_row:
+                if prev_row and curr_row:
+                    prev_key_val = prev_row[key]
+                    curr_key_val = curr_row[key]
+                    
+                    if prev_key_val == curr_key_val:
+                        # Check for changes
+                        changed_fields = {
+                            col: [prev_row.get(col), curr_row.get(col)]
+                            for col in compare_columns
+                            if col in prev_columns and col in curr_columns and prev_row.get(col) != curr_row.get(col)
+                        }
+                        if changed_fields:
+                            result["changed"].append({
+                                "key": prev_key_val,
+                                "changes": changed_fields
+                            })
+                        prev_row = next(reader1, None)
+                        curr_row = next(reader2, None)
+                    elif prev_key_val < curr_key_val:
+                        # Row removed
+                        result["removed"].append(prev_row)
+                        prev_row = next(reader1, None)
+                    else:
+                        # Row added
+                        result["added"].append(curr_row)
+                        curr_row = next(reader2, None)
+                elif prev_row:
                     result["removed"].append(prev_row)
                     prev_row = next(reader1, None)
-                else:
-                    # Row added
+                elif curr_row:
                     result["added"].append(curr_row)
                     curr_row = next(reader2, None)
-            elif prev_row:
-                result["removed"].append(prev_row)
-                prev_row = next(reader1, None)
-            elif curr_row:
-                result["added"].append(curr_row)
-                curr_row = next(reader2, None)
+    except (csv.Error, UnicodeDecodeError) as e:
+        logger.error(f"CSV processing error: {str(e)}")
+        raise
+        
     return result
 
 def human_text(result, key=None, current=None, extras=None):
     """
     Render the diff result as a human-readable string.
+    
+    Args:
+        result: Diff result dictionary
+        key: Name of the key column (optional)
+        current: Current data dictionary (optional)
+        extras: Extra fields to display (optional)
+        
+    Returns:
+        Formatted string with diff information
     """
     title = []
     summary = []
     show_headers = sum(1 for k in result if result[k]) > 1
+    
+    # Handle column changes
     if result["columns_added"]:
         fragment = f"{len(result['columns_added'])} {'column' if len(result['columns_added']) == 1 else 'columns'} added"
         title.append(fragment)
@@ -213,6 +332,8 @@ def human_text(result, key=None, current=None, extras=None):
             + [f"  {c}" for c in sorted(result["columns_removed"])]
             + [""]
         )
+        
+    # Handle changed rows
     if result["changed"]:
         fragment = f"{len(result['changed'])} rows changed"
         title.append(fragment)
@@ -224,9 +345,10 @@ def human_text(result, key=None, current=None, extras=None):
             block.append(f"  {key}: {details['key']}")
             for field, (prev_value, current_value) in details["changes"].items():
                 block.append(f'    {field}: "{prev_value}" => "{current_value}"')
-            if extras:
-                current_item = current[details["key"]]
-                block.append(human_extras(current_item, extras))
+            if extras and current:
+                current_item = current.get(details["key"])
+                if current_item:
+                    block.append(human_extras(current_item, extras))
             block.append("")
             change_blocks.append("\n".join(block))
             if details.get("unchanged"):
@@ -237,6 +359,8 @@ def human_text(result, key=None, current=None, extras=None):
                 block.append("")
                 change_blocks.append("\n".join(block))
         summary.append("\n".join(change_blocks))
+        
+    # Handle added rows
     if result["added"]:
         fragment = f"{len(result['added'])} rows added"
         title.append(fragment)
@@ -250,6 +374,8 @@ def human_text(result, key=None, current=None, extras=None):
             rows.append(to_append)
         summary.append("\n\n".join(rows))
         summary.append("")
+        
+    # Handle removed rows
     if result["removed"]:
         fragment = f"{len(result['removed'])} rows removed"
         title.append(fragment)
@@ -263,19 +389,37 @@ def human_text(result, key=None, current=None, extras=None):
             rows.append(to_append)
         summary.append("\n\n".join(rows))
         summary.append("")
+        
     return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
 
 def human_row(row, prefix=""):
     """
     Render a row as a human-readable string.
+    
+    Args:
+        row: Row data dictionary
+        prefix: String prefix for each line
+        
+    Returns:
+        Formatted string representation of the row
     """
     return "\n".join(f"{prefix}{key}: {value}" for key, value in row.items())
 
 def human_extras(row, extras):
     """
     Render extra fields for a row.
+    
+    Args:
+        row: Row data dictionary
+        extras: List of (key, format) tuples
+        
+    Returns:
+        Formatted string with extra information
     """
     bits = ["  extras:"]
     for key, fmt in extras:
-        bits.append(f"    {key}: {fmt.format(**row)}")
+        try:
+            bits.append(f"    {key}: {fmt.format(**row)}")
+        except (KeyError, ValueError) as e:
+            bits.append(f"    {key}: <error formatting: {str(e)}>")
     return "\n".join(bits)
diff --git a/db_diff/cli.py b/db_diff/cli.py
index db20f15..6968cd5 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -1,23 +1,48 @@
+"""
+Command-line interface for DB-Diff tool.
+
+This module provides the command-line interface for comparing CSV and JSON files.
+"""
+
 import click
 import json as std_json
 import time as time_module
 import os
 import datetime
 import csv
-import shutil  # Add this import
-from . import load_csv, load_json, compare, human_text, streaming_compare_csv
+import shutil
+import logging
+from . import load_csv, load_json, compare, human_text, streaming_compare_csv, logger
+
+# Configure logging with click integration
+logger = logging.getLogger("db_diff")
 
 # Get terminal width for better help formatting
 TERMINAL_WIDTH = shutil.get_terminal_size().columns
 
+# Known CSV dialects
 DIALECTS = {
     "csv": "excel",
     "tsv": "excel-tab",
 }
 
+# Supported input formats
+SUPPORTED_FORMATS = ("csv", "tsv", "json")
+
 def sniff_key_column(filename, input_format, encoding="utf-8"):
     """
     Detect the key column from the first row of a CSV/TSV/JSON file.
+    
+    Args:
+        filename: Path to the input file
+        input_format: Format of the input file (csv, tsv, json)
+        encoding: File encoding (default: utf-8)
+        
+    Returns:
+        The name of the first column or key
+        
+    Raises:
+        click.ClickException: If key detection fails
     """
     try:
         if input_format == "json":
@@ -29,7 +54,10 @@ def sniff_key_column(filename, input_format, encoding="utf-8"):
             with open(filename, newline="", encoding=encoding) as f:
                 reader = csv.reader(f, dialect=DIALECTS.get(input_format))
                 headings = next(reader)
-                return headings[0]
+                if headings:
+                    return headings[0]
+                else:
+                    raise click.ClickException("No header row found in CSV/TSV file")
     except Exception as e:
         raise click.ClickException(f"Could not sniff key column: {e}")
     return None
@@ -37,6 +65,18 @@ def sniff_key_column(filename, input_format, encoding="utf-8"):
 def load(filename, key, input_format, encoding="utf-8"):
     """
     Load a CSV/TSV/JSON file into a dictionary keyed by the given column.
+    
+    Args:
+        filename: Path to the input file
+        key: Column name to use as key
+        input_format: Format of the input file (csv, tsv, json)
+        encoding: File encoding (default: utf-8)
+        
+    Returns:
+        Dictionary of data rows keyed by the specified column
+        
+    Raises:
+        click.ClickException: If loading fails
     """
     try:
         if input_format == "json":
@@ -48,6 +88,43 @@ def load(filename, key, input_format, encoding="utf-8"):
     except Exception as e:
         raise click.ClickException(f"Failed to load '{filename}': {e}")
 
+def generate_output_filename(previous, current, output_file, output_path, timestamp=None):
+    """
+    Generate an output filename for the diff result.
+    
+    Args:
+        previous: Path to the previous file
+        current: Path to the current file
+        output_file: User-specified output filename (optional)
+        output_path: User-specified output directory (optional)
+        timestamp: Timestamp to use (optional)
+        
+    Returns:
+        Full path to the output file
+    """
+    if timestamp is None:
+        timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    
+    # Extract base filenames without extensions for the comparison files
+    prev_basename = os.path.splitext(os.path.basename(previous))[0]
+    curr_basename = os.path.splitext(os.path.basename(current))[0]
+    
+    if not output_file:
+        # Default output filename
+        output_file = f"{timestamp}_diffs_{prev_basename}__VS__{curr_basename}.json"
+    else:
+        # Append the user-specified filename to the timestamp
+        base_name = os.path.splitext(output_file)[0]  # Remove extension if present
+        output_file = f"{timestamp}_diffs_{base_name}.json"
+    
+    # Handle output path
+    if output_path:
+        if not os.path.isdir(output_path):
+            raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
+        output_file = os.path.join(output_path, os.path.basename(output_file))
+        
+    return output_file
+
 @click.command(
     context_settings={
         "help_option_names": ["-h", "--help"],
@@ -75,6 +152,7 @@ def load(filename, key, input_format, encoding="utf-8"):
     "--output",
     default="readable",
     show_default=True,
+    type=click.Choice(["readable", "json", "jsonfile"], case_sensitive=False),
     help="Output format. Available: (readable|json|jsonfile).",
 )
 @click.option(
@@ -113,6 +191,7 @@ def load(filename, key, input_format, encoding="utf-8"):
     "--format",
     "input_format",
     default=None,
+    type=click.Choice(list(SUPPORTED_FORMATS), case_sensitive=False),
     help="Explicitly specify input format. Available (csv|tsv|json). [default: auto-detect based on file extension]",
 )
 @click.option(
@@ -132,21 +211,52 @@ def load(filename, key, input_format, encoding="utf-8"):
     is_flag=True,
     help="List available fields/columns in the input files and exit.",
 )
+@click.option(
+    "--verbose", "-v",
+    is_flag=True,
+    help="Enable verbose output.",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,
     show_unchanged, encoding, output, output_file, output_path,
-    fields, ignorefields, streaming, listfields
+    fields, ignorefields, streaming, listfields, verbose
 ):
     """
     Compare the differences between two CSV or JSON files.
+    
+    Args:
+        previous: Path to the previous file
+        current: Path to the current file
+        key: Column name to use as key (optional)
+        input_format: Format of the input files (optional)
+        show_unchanged: Whether to show unchanged fields
+        encoding: File encoding
+        output: Output format
+        output_file: Output filename (optional)
+        output_path: Output directory (optional)
+        fields: List of fields to compare (optional)
+        ignorefields: List of fields to ignore (optional)
+        streaming: Whether to use streaming mode
+        listfields: Whether to list available fields
+        verbose: Whether to enable verbose output
     """
+    # Set logging level based on verbosity
+    if verbose:
+        logger.setLevel(logging.DEBUG)
+        click.echo("Verbose mode enabled")
+    else:
+        logger.setLevel(logging.INFO)
+    
     click.echo(f"Starting diff: '{previous}' vs '{current}'...")
 
     # Detect file format
-    detected_format = input_format or previous.split(".")[-1].lower()
-    if detected_format not in ("csv", "tsv", "json"):
-        raise click.ClickException(f"Unsupported file format: {detected_format}")
+    detected_format = input_format or os.path.splitext(previous)[1].lower().lstrip('.')
+    if detected_format not in SUPPORTED_FORMATS:
+        raise click.ClickException(
+            f"Unsupported file format: {detected_format}. "
+            f"Supported formats are: {', '.join(SUPPORTED_FORMATS)}"
+        )
 
     # List fields and exit
     if listfields:
@@ -156,45 +266,37 @@ def cli(
                     data = std_json.load(f)
                     if isinstance(data, list) and data:
                         click.echo("Available fields:")
-                        for field in data[0].keys():
+                        for field in sorted(data[0].keys()):
                             click.echo(f"  {field}")
+                    else:
+                        click.echo("No fields found: JSON file does not contain a list of objects")
             else:
                 with open(previous, newline="", encoding=encoding) as f:
                     reader = csv.reader(f, dialect=DIALECTS.get(detected_format))
                     headings = next(reader)
-                    click.echo("Available fields:")
-                    for field in headings:
-                        click.echo(f"  {field}")
+                    if headings:
+                        click.echo("Available fields:")
+                        for field in headings:
+                            click.echo(f"  {field}")
+                    else:
+                        click.echo("No fields found: CSV file appears to be empty")
         except Exception as e:
             raise click.ClickException(f"Could not list fields: {e}")
         return
 
     # Set default output filename if needed
     if output == "jsonfile":
-        timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-        
-        # Extract base filenames without extensions for the comparison files
-        prev_basename = os.path.splitext(os.path.basename(previous))[0]
-        curr_basename = os.path.splitext(os.path.basename(current))[0]
-        default_name = f"{timestamp}_diffs_{prev_basename}__VS__{curr_basename}.json"
-        
-        if not output_file:
-            output_file = default_name
-        else:
-            # Append the user-specified filename to the timestamp
-            base_name = os.path.splitext(output_file)[0]  # Remove extension if present
-            output_file = f"{timestamp}_diffs_{base_name}.json"
-        
-        if output_path:
-            if not os.path.isdir(output_path):
-                raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
-            output_file = os.path.join(output_path, os.path.basename(output_file))
+        try:
+            output_file = generate_output_filename(previous, current, output_file, output_path)
+        except Exception as e:
+            raise click.ClickException(f"Failed to generate output filename: {e}")
 
     # Detect key column before loading data
     if key is None:
         key = sniff_key_column(previous, detected_format, encoding)
         if not key:
             raise click.ClickException("Could not determine key column from the previous file.")
+        click.echo(f"Using '{key}' as the key column")
 
     # Enforce mutual exclusivity
     if fields and ignorefields:
@@ -204,48 +306,80 @@ def cli(
     fields_set = set(f.strip() for f in fields.split(",")) if fields else None
     ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
 
-    # Always measure time (removed the --time option)
+    # Always measure time
     start = time_module.time()
 
-    # --- Streaming logic ---
-    if (streaming or (detected_format in ("csv", "tsv") and key)) and not fields:
-        if detected_format not in ("csv", "tsv"):
-            raise click.ClickException("--streaming only works with CSV/TSV files.")
-        diff = streaming_compare_csv(
-            previous,
-            current,
-            key=key,
-            compare_columns=fields_set,
-            ignorefields=ignorefields_set,
-            encoding=encoding,
-            dialect=DIALECTS.get(detected_format, "excel"),
-        )
-        # For human_text, we need current_data for extras (if used)
-        current_data = None
-    else:
-        previous_data = load(previous, key, detected_format, encoding)
-        current_data = load(current, key, detected_format, encoding)
-        diff = compare(
-            previous_data,
-            current_data,
-            show_unchanged,
-            fields=fields_set,
-            ignorefields=ignorefields_set,
-        )
+    try:
+        # --- Streaming logic ---
+        if (streaming or (detected_format in ("csv", "tsv") and key)) and not fields:
+            if detected_format not in ("csv", "tsv"):
+                raise click.ClickException("--streaming only works with CSV/TSV files.")
+                
+            click.echo("Using streaming mode for comparison...")
+            diff = streaming_compare_csv(
+                previous,
+                current,
+                key=key,
+                compare_columns=fields_set,
+                ignorefields=ignorefields_set,
+                encoding=encoding,
+                dialect=DIALECTS.get(detected_format, "excel"),
+            )
+            # For human_text, we need current_data for extras (if used)
+            current_data = None
+        else:
+            click.echo("Loading previous data...")
+            previous_data = load(previous, key, detected_format, encoding)
+            click.echo(f"Loaded {len(previous_data)} rows from previous file")
+            
+            click.echo("Loading current data...")
+            current_data = load(current, key, detected_format, encoding)
+            click.echo(f"Loaded {len(current_data)} rows from current file")
+            
+            click.echo("Comparing data...")
+            diff = compare(
+                previous_data,
+                current_data,
+                show_unchanged,
+                fields=fields_set,
+                ignorefields=ignorefields_set,
+            )
 
-    if output == "json":
-        print(std_json.dumps(diff, indent=4))
-        click.echo("Finished. Output printed as JSON to terminal.")
-    elif output == "jsonfile":
-        with open(output_file, "w", encoding=encoding) as f:
-            f.write(std_json.dumps(diff, indent=4))
-        click.echo(f"Finished. Output saved to file: {output_file}")
-    else:
-        print(
-            human_text(diff, key, current=current_data)
-        )
-        click.echo("Finished. Output printed in readable format to terminal.")
+        # Generate output
+        if output == "json":
+            print(std_json.dumps(diff, indent=4))
+            click.echo("Finished. Output printed as JSON to terminal.")
+        elif output == "jsonfile":
+            with open(output_file, "w", encoding=encoding) as f:
+                f.write(std_json.dumps(diff, indent=4))
+            click.echo(f"Finished. Output saved to file: {output_file}")
+        else:
+            print(
+                human_text(diff, key, current=current_data)
+            )
+            click.echo("Finished. Output printed in readable format to terminal.")
+    
+    except Exception as e:
+        logger.error(f"Error during comparison: {str(e)}")
+        if verbose:
+            import traceback
+            logger.debug(traceback.format_exc())
+        raise click.ClickException(str(e))
 
     # Always display the elapsed time
     end = time_module.time()
-    print(f"\nElapsed time: {end - start:.3f} seconds")
+    click.echo(f"\nElapsed time: {end - start:.3f} seconds")
+    
+    # Display summary statistics
+    summary = []
+    for key in ["columns_added", "columns_removed", "added", "removed", "changed"]:
+        if diff[key]:
+            summary.append(f"{len(diff[key])} {key.replace('_', ' ')}")
+    
+    if summary:
+        click.echo(f"Summary: {', '.join(summary)}")
+    else:
+        click.echo("No differences found")
+
+if __name__ == "__main__":
+    cli()
diff --git a/setup.py b/setup.py
index bc3b8b3..02e6b39 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "4.0"
+VERSION = "4.1"
 
 def get_long_description():
     with io.open(
@@ -21,8 +21,8 @@ def get_long_description():
     license="Apache License, Version 2.0",
     packages=find_packages(),
     install_requires=[
-        "click>=7.0",
-        "dictdiffer>=0.8.0",
+        "click>=8.0.0",
+        "dictdiffer>=0.9.0",
         "xxhash>=3.0.0"
     ],
     entry_points={

From 43e4392b8d043f1fef740bac4fefade51bd96bdb Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:31:16 -0500
Subject: [PATCH 092/129] removed old, unused code

---
 db_diff/__init__.py | 89 ++++++++++++++++++++++-----------------------
 db_diff/cli.py      | 24 ++++++------
 2 files changed, 54 insertions(+), 59 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index b497690..96b58b7 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -17,6 +17,18 @@
 
 __version__ = "1.0.0"
 
+def _generate_hash_key(row):
+    """
+    Generate a hash key for a row.
+    
+    Args:
+        row: Row data dictionary
+        
+    Returns:
+        Hash key as a string
+    """
+    return xxhash.xxh64(json.dumps(row, sort_keys=True).encode("utf8")).hexdigest()
+
 def load_csv(fp, key=None, dialect=None):
     """
     Load a CSV file into a dictionary keyed by the given column or hash.
@@ -45,7 +57,7 @@ def load_csv(fp, key=None, dialect=None):
     if key:
         keyfn = lambda r: r[key]
     else:
-        keyfn = lambda r: xxhash.xxh64(json.dumps(r, sort_keys=True).encode("utf8")).hexdigest()
+        keyfn = _generate_hash_key
     return {keyfn(r): r for r in rows}
 
 def load_json(fp, key=None):
@@ -79,7 +91,7 @@ def load_json(fp, key=None):
     if key:
         keyfn = lambda r: r.get(key, str(id(r)))
     else:
-        keyfn = lambda r: xxhash.xxh64(json.dumps(r, sort_keys=True).encode("utf8")).hexdigest()
+        keyfn = _generate_hash_key
         
     return {keyfn(r): _simplify_json_row(r, common_keys) for r in raw_list}
 
@@ -108,6 +120,31 @@ def _simplify_json_row(r, common_keys):
             
     return result
 
+def _determine_columns_to_compare(prev_columns, curr_columns, fields=None, ignorefields=None):
+    """
+    Determine which columns to compare based on fields and ignorefields.
+    
+    Args:
+        prev_columns: Set of column names from previous data
+        curr_columns: Set of column names from current data
+        fields: List of specific fields to compare (optional)
+        ignorefields: List of fields to ignore in comparison (optional)
+        
+    Returns:
+        Tuple of (compare_columns, ignore_columns)
+    """
+    if fields:
+        compare_columns = set(fields)
+    elif ignorefields:
+        compare_columns = prev_columns | curr_columns
+        compare_columns -= set(ignorefields)
+    else:
+        compare_columns = prev_columns | curr_columns
+        
+    ignore_columns = (prev_columns | curr_columns) - compare_columns
+    
+    return compare_columns, ignore_columns
+
 def compare(previous, current, show_unchanged=False, fields=None, ignorefields=None):
     """
     Compare two dictionaries of rows and return a diff summary.
@@ -143,18 +180,12 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
     current_columns = set(next(iter(current.values())).keys())
 
     # Apply fields/ignorefields filtering
-    if fields:
-        compare_columns = set(fields)
-    elif ignorefields:
-        compare_columns = previous_columns | current_columns
-        compare_columns -= set(ignorefields)
-    else:
-        compare_columns = previous_columns | current_columns
+    compare_columns, ignore_columns = _determine_columns_to_compare(previous_columns, current_columns, fields, ignorefields)
 
     # Adjust columns_added/removed based on compare_columns
     result["columns_added"] = [c for c in current_columns if c not in previous_columns and c in compare_columns]
     result["columns_removed"] = [c for c in previous_columns if c not in current_columns and c in compare_columns]
-    ignore_columns = (previous_columns | current_columns) - compare_columns
+    #ignore_columns = (previous_columns | current_columns) - compare_columns
 
     # Have any rows been removed or added?
     removed = [id for id in previous if id not in current]
@@ -249,12 +280,7 @@ def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignor
             curr_columns = set(reader2.fieldnames)
 
             # Determine columns to compare
-            if compare_columns:
-                compare_columns = set(compare_columns)
-            elif ignorefields:
-                compare_columns = (prev_columns | curr_columns) - set(ignorefields)
-            else:
-                compare_columns = prev_columns | curr_columns
+            compare_columns, _ = _determine_columns_to_compare(prev_columns, curr_columns, compare_columns, ignorefields)
 
             result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns]
             result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns]
@@ -298,15 +324,13 @@ def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignor
         
     return result
 
-def human_text(result, key=None, current=None, extras=None):
+def human_text(result, key=None):
     """
     Render the diff result as a human-readable string.
     
     Args:
         result: Diff result dictionary
         key: Name of the key column (optional)
-        current: Current data dictionary (optional)
-        extras: Extra fields to display (optional)
         
     Returns:
         Formatted string with diff information
@@ -345,10 +369,6 @@ def human_text(result, key=None, current=None, extras=None):
             block.append(f"  {key}: {details['key']}")
             for field, (prev_value, current_value) in details["changes"].items():
                 block.append(f'    {field}: "{prev_value}" => "{current_value}"')
-            if extras and current:
-                current_item = current.get(details["key"])
-                if current_item:
-                    block.append(human_extras(current_item, extras))
             block.append("")
             change_blocks.append("\n".join(block))
             if details.get("unchanged"):
@@ -369,8 +389,6 @@ def human_text(result, key=None, current=None, extras=None):
         rows = []
         for row in result["added"]:
             to_append = human_row(row, prefix="  ")
-            if extras:
-                to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
         summary.append("\n\n".join(rows))
         summary.append("")
@@ -384,8 +402,6 @@ def human_text(result, key=None, current=None, extras=None):
         rows = []
         for row in result["removed"]:
             to_append = human_row(row, prefix="  ")
-            if extras:
-                to_append += "\n" + human_extras(row, extras)
             rows.append(to_append)
         summary.append("\n\n".join(rows))
         summary.append("")
@@ -404,22 +420,3 @@ def human_row(row, prefix=""):
         Formatted string representation of the row
     """
     return "\n".join(f"{prefix}{key}: {value}" for key, value in row.items())
-
-def human_extras(row, extras):
-    """
-    Render extra fields for a row.
-    
-    Args:
-        row: Row data dictionary
-        extras: List of (key, format) tuples
-        
-    Returns:
-        Formatted string with extra information
-    """
-    bits = ["  extras:"]
-    for key, fmt in extras:
-        try:
-            bits.append(f"    {key}: {fmt.format(**row)}")
-        except (KeyError, ValueError) as e:
-            bits.append(f"    {key}: <error formatting: {str(e)}>")
-    return "\n".join(bits)
diff --git a/db_diff/cli.py b/db_diff/cli.py
index 6968cd5..a6804cd 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -5,8 +5,8 @@
 """
 
 import click
-import json as std_json
-import time as time_module
+import json
+import time
 import os
 import datetime
 import csv
@@ -47,7 +47,7 @@ def sniff_key_column(filename, input_format, encoding="utf-8"):
     try:
         if input_format == "json":
             with open(filename, encoding=encoding) as f:
-                data = std_json.load(f)
+                data = json.load(f)
                 if isinstance(data, list) and data:
                     return list(data[0].keys())[0]
         else:
@@ -173,13 +173,13 @@ def generate_output_filename(previous, current, output_file, output_path, timest
     "--fields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age__c,Email__c).",
+    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age,Email).",
 )
 @click.option(
     "--ignorefields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to ignore during comparison. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp).",
+    help="Comma-separated list of fields to ignore. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp).",
 )
 @click.option(
     "--showunchanged",
@@ -263,7 +263,7 @@ def cli(
         try:
             if detected_format == "json":
                 with open(previous, encoding=encoding) as f:
-                    data = std_json.load(f)
+                    data = json.load(f)
                     if isinstance(data, list) and data:
                         click.echo("Available fields:")
                         for field in sorted(data[0].keys()):
@@ -307,7 +307,7 @@ def cli(
     ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
 
     # Always measure time
-    start = time_module.time()
+    start = time.time()
 
     try:
         # --- Streaming logic ---
@@ -325,8 +325,6 @@ def cli(
                 encoding=encoding,
                 dialect=DIALECTS.get(detected_format, "excel"),
             )
-            # For human_text, we need current_data for extras (if used)
-            current_data = None
         else:
             click.echo("Loading previous data...")
             previous_data = load(previous, key, detected_format, encoding)
@@ -347,15 +345,15 @@ def cli(
 
         # Generate output
         if output == "json":
-            print(std_json.dumps(diff, indent=4))
+            print(json.dumps(diff, indent=4))
             click.echo("Finished. Output printed as JSON to terminal.")
         elif output == "jsonfile":
             with open(output_file, "w", encoding=encoding) as f:
-                f.write(std_json.dumps(diff, indent=4))
+                f.write(json.dumps(diff, indent=4))
             click.echo(f"Finished. Output saved to file: {output_file}")
         else:
             print(
-                human_text(diff, key, current=current_data)
+                human_text(diff, key)
             )
             click.echo("Finished. Output printed in readable format to terminal.")
     
@@ -367,7 +365,7 @@ def cli(
         raise click.ClickException(str(e))
 
     # Always display the elapsed time
-    end = time_module.time()
+    end = time.time()
     click.echo(f"\nElapsed time: {end - start:.3f} seconds")
     
     # Display summary statistics

From cdf26247ec2b9683c1f2e07945355f8b1243500c Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:31:37 -0500
Subject: [PATCH 093/129] minor

---
 db_diff/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 96b58b7..5eca158 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -1,5 +1,5 @@
 """
-DB-Diff - A tool for comparing database dumps in CSV or JSON format.
+db-diff - A tool for comparing database dumps in CSV or JSON format.
 
 This module provides functions for loading and comparing data from CSV and JSON files,
 generating diffs, and displaying results in human-readable format.

From d86f604a782d30341275c20cf8440929e6ac085d Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:37:22 -0500
Subject: [PATCH 094/129] minor

---
 db_diff/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index 5eca158..ff4d9d0 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -15,8 +15,6 @@
 logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
 logger = logging.getLogger(__name__)
 
-__version__ = "1.0.0"
-
 def _generate_hash_key(row):
     """
     Generate a hash key for a row.

From 0c97cc12b5896b74c208f3b78b44404d558e2a5c Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:47:26 -0500
Subject: [PATCH 095/129] updated readme

---
 README.md | 40 ++++++++++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 7964ae5..248dbd1 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,9 @@
 - [Examples](#examples)
 - [Performance](#performance)
 - [Development](#development)
+- [Contributing](#contributing)
+- [Compatibility](#compatibility)
+- [Changelog](#changelog)
 - [License](#license)
 
 ---
@@ -40,7 +43,7 @@
 
 ## Installation
 
-Install the latest version directly from GitHub:
+### From GitHub (Latest Development Version)
 
 ```sh
 pip install git+https://github.com/datsom1/db-diff.git
@@ -248,11 +251,11 @@ A summary of key options:
 | `--fields TEXT`       | Comma-separated list of fields to compare (all others ignored)                               |
 | `--ignorefields TEXT` | Comma-separated list of fields to ignore during comparison                                   |
 | `--showunchanged`     | Show all fields for changed records, not just changed fields                                 |
-| `--time`              | Measure and display elapsed time for the diff operation                                      |
 | `--format TEXT`       | Explicitly specify input format: `csv`, `tsv`, or `json` (default: auto-detect)              |
 | `--encoding TEXT`     | Input file encoding (default: utf-8)                                                         |
 | `--streaming`         | Use streaming mode for very large CSV/TSV files (requires files to be sorted by key)         |
 | `--listfields`        | List available fields/columns in the input files and exit                                    |
+| `--verbose`, `-v`     | Enable verbose output with detailed logging                                                  |
 | `--version`           | Show the version and exit                                                                    |
 | `-h, --help`          | Show help message and exit                                                                   |
 
@@ -296,6 +299,12 @@ db-diff one.csv two.csv --key=Id --ignorefields=LastModifiedDate
 db-diff large1.csv large2.csv --key=Id --streaming
 ```
 
+**Verbose output with detailed logging:**
+
+```sh
+db-diff one.csv two.csv --key=Id --verbose
+```
+
 ---
 
 ## Performance
@@ -314,13 +323,31 @@ git clone https://github.com/datsom1/db-diff.git
 cd db-diff
 ```
 
-Install dependencies:
+Install development dependencies:
 
 ```sh
-pip install -e .
+pip install -e ".[dev]"
 ```
 
----
+Run tests:
+
+```sh
+pytest
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## Compatibility
+
+- Python 3.6+
+- Works on Windows, macOS, and Linux
+- Handles various file encodings (UTF-8, UTF-16, Latin1, etc.)
+
+## Changelog
+
+For a detailed list of changes for each version, please see the [CHANGELOG](https://github.com/datsom1/db-diff/releases).
 
 ## License
 
@@ -329,4 +356,5 @@ This project is licensed under the [Apache License 2.0](LICENSE).
 ---
 
 **Author:** Thomas Coyle  
-**Repository:** https://github.com/datsom1/db-diff
+**Repository:** https://github.com/datsom1/db-diff  
+**Last Updated:** June 2025

From 26fd217d474c891a50c710c5f9b3b9a19fb59fc4 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:48:02 -0500
Subject: [PATCH 096/129] update

---
 README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 248dbd1..74e2483 100644
--- a/README.md
+++ b/README.md
@@ -356,5 +356,4 @@ This project is licensed under the [Apache License 2.0](LICENSE).
 ---
 
 **Author:** Thomas Coyle  
-**Repository:** https://github.com/datsom1/db-diff  
-**Last Updated:** June 2025
+**Repository:** https://github.com/datsom1/db-diff  
\ No newline at end of file

From 33cd0b706193cca7fed7b3ea817e974c4a13e0af Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:49:34 -0500
Subject: [PATCH 097/129] Update README.md

---
 README.md | 41 +----------------------------------------
 1 file changed, 1 insertion(+), 40 deletions(-)

diff --git a/README.md b/README.md
index 74e2483..84d3e43 100644
--- a/README.md
+++ b/README.md
@@ -18,10 +18,6 @@
 - [Options](#options)
 - [Examples](#examples)
 - [Performance](#performance)
-- [Development](#development)
-- [Contributing](#contributing)
-- [Compatibility](#compatibility)
-- [Changelog](#changelog)
 - [License](#license)
 
 ---
@@ -314,41 +310,6 @@ db-diff one.csv two.csv --key=Id --verbose
 
 ---
 
-## Development
-
-Clone the repository:
-
-```sh
-git clone https://github.com/datsom1/db-diff.git
-cd db-diff
-```
-
-Install development dependencies:
-
-```sh
-pip install -e ".[dev]"
-```
-
-Run tests:
-
-```sh
-pytest
-```
-
-## Contributing
-
-Contributions are welcome! Please feel free to submit a Pull Request.
-
-## Compatibility
-
-- Python 3.6+
-- Works on Windows, macOS, and Linux
-- Handles various file encodings (UTF-8, UTF-16, Latin1, etc.)
-
-## Changelog
-
-For a detailed list of changes for each version, please see the [CHANGELOG](https://github.com/datsom1/db-diff/releases).
-
 ## License
 
 This project is licensed under the [Apache License 2.0](LICENSE).
@@ -356,4 +317,4 @@ This project is licensed under the [Apache License 2.0](LICENSE).
 ---
 
 **Author:** Thomas Coyle  
-**Repository:** https://github.com/datsom1/db-diff  
\ No newline at end of file
+**Repository:** https://github.com/datsom1/db-diff  

From 35c9b89a64e620df6425c10222d015914bd6a94a Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 14:50:08 -0500
Subject: [PATCH 098/129] Update README.md

---
 README.md | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/README.md b/README.md
index 84d3e43..469553c 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,6 @@
 - [Options](#options)
 - [Examples](#examples)
 - [Performance](#performance)
-- [License](#license)
 
 ---
 
@@ -310,11 +309,5 @@ db-diff one.csv two.csv --key=Id --verbose
 
 ---
 
-## License
-
-This project is licensed under the [Apache License 2.0](LICENSE).
-
----
-
 **Author:** Thomas Coyle  
 **Repository:** https://github.com/datsom1/db-diff  

From a7f73aeb4045b113692433bfecf82237c15178c0 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 15:01:49 -0500
Subject: [PATCH 099/129] improvements

---
 db_diff/cli.py | 128 ++++++++++++++++++++++++++++++++++++-------------
 setup.py       |   3 +-
 2 files changed, 96 insertions(+), 35 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index a6804cd..704bdfc 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -4,15 +4,26 @@
 This module provides the command-line interface for comparing CSV and JSON files.
 """
 
-import click
-import json
-import time
 import os
-import datetime
 import csv
+import json
+import time
 import shutil
 import logging
-from . import load_csv, load_json, compare, human_text, streaming_compare_csv, logger
+import datetime
+import traceback
+from typing import Dict, Set, List, Optional, Any
+
+import click
+
+from . import (
+    load_csv, 
+    load_json, 
+    compare, 
+    human_text, 
+    streaming_compare_csv, 
+    logger
+)
 
 # Configure logging with click integration
 logger = logging.getLogger("db_diff")
@@ -29,7 +40,7 @@
 # Supported input formats
 SUPPORTED_FORMATS = ("csv", "tsv", "json")
 
-def sniff_key_column(filename, input_format, encoding="utf-8"):
+def sniff_key_column(filename: str, input_format: str, encoding: str = "utf-8") -> Optional[str]:
     """
     Detect the key column from the first row of a CSV/TSV/JSON file.
     
@@ -39,30 +50,37 @@ def sniff_key_column(filename, input_format, encoding="utf-8"):
         encoding: File encoding (default: utf-8)
         
     Returns:
-        The name of the first column or key
+        The name of the first column or key, or None if detection fails
         
     Raises:
         click.ClickException: If key detection fails
     """
+    logger.debug(f"Sniffing key column from {filename} with format {input_format}")
     try:
         if input_format == "json":
             with open(filename, encoding=encoding) as f:
                 data = json.load(f)
                 if isinstance(data, list) and data:
-                    return list(data[0].keys())[0]
+                    first_key = list(data[0].keys())[0]
+                    logger.debug(f"Found first key in JSON: {first_key}")
+                    return first_key
+                else:
+                    raise click.ClickException("JSON file does not contain a list of objects with keys")
         else:
             with open(filename, newline="", encoding=encoding) as f:
-                reader = csv.reader(f, dialect=DIALECTS.get(input_format))
-                headings = next(reader)
-                if headings:
+                reader = csv.reader(f, dialect=DIALECTS.get(input_format, "excel"))
+                headings = next(reader, None)
+                if headings and len(headings) > 0:
+                    logger.debug(f"Found first column in CSV/TSV: {headings[0]}")
                     return headings[0]
                 else:
                     raise click.ClickException("No header row found in CSV/TSV file")
     except Exception as e:
+        logger.error(f"Error sniffing key column: {str(e)}")
         raise click.ClickException(f"Could not sniff key column: {e}")
     return None
 
-def load(filename, key, input_format, encoding="utf-8"):
+def load(filename: str, key: str, input_format: str, encoding: str = "utf-8") -> Dict:
     """
     Load a CSV/TSV/JSON file into a dictionary keyed by the given column.
     
@@ -78,17 +96,24 @@ def load(filename, key, input_format, encoding="utf-8"):
     Raises:
         click.ClickException: If loading fails
     """
+    logger.debug(f"Loading file {filename} with format {input_format}, key={key}")
     try:
         if input_format == "json":
             with open(filename, encoding=encoding) as f:
-                return load_json(f, key=key)
+                data = load_json(f, key=key)
+                logger.debug(f"Loaded {len(data)} rows from JSON file")
+                return data
         else:
             with open(filename, newline="", encoding=encoding) as f:
-                return load_csv(f, key=key, dialect=DIALECTS.get(input_format))
+                data = load_csv(f, key=key, dialect=DIALECTS.get(input_format, "excel"))
+                logger.debug(f"Loaded {len(data)} rows from CSV/TSV file")
+                return data
     except Exception as e:
+        logger.error(f"Error loading file {filename}: {str(e)}")
         raise click.ClickException(f"Failed to load '{filename}': {e}")
 
-def generate_output_filename(previous, current, output_file, output_path, timestamp=None):
+def generate_output_filename(previous: str, current: str, output_file: Optional[str], 
+                         output_path: Optional[str], timestamp: Optional[str] = None) -> str:
     """
     Generate an output filename for the diff result.
     
@@ -101,6 +126,9 @@ def generate_output_filename(previous, current, output_file, output_path, timest
         
     Returns:
         Full path to the output file
+        
+    Raises:
+        click.ClickException: If the output path doesn't exist
     """
     if timestamp is None:
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
@@ -120,9 +148,11 @@ def generate_output_filename(previous, current, output_file, output_path, timest
     # Handle output path
     if output_path:
         if not os.path.isdir(output_path):
+            logger.error(f"Output path '{output_path}' does not exist")
             raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
         output_file = os.path.join(output_path, os.path.basename(output_file))
-        
+    
+    logger.debug(f"Generated output filename: {output_file}")
     return output_file
 
 @click.command(
@@ -265,22 +295,28 @@ def cli(
                 with open(previous, encoding=encoding) as f:
                     data = json.load(f)
                     if isinstance(data, list) and data:
-                        click.echo("Available fields:")
-                        for field in sorted(data[0].keys()):
+                        all_fields = set()
+                        # Collect all unique fields across the first 10 records
+                        for i, item in enumerate(data[:10]):
+                            all_fields.update(item.keys())
+                        
+                        click.echo(f"Available fields in '{os.path.basename(previous)}' ({len(all_fields)} total):")
+                        for field in sorted(all_fields):
                             click.echo(f"  {field}")
                     else:
                         click.echo("No fields found: JSON file does not contain a list of objects")
             else:
                 with open(previous, newline="", encoding=encoding) as f:
-                    reader = csv.reader(f, dialect=DIALECTS.get(detected_format))
-                    headings = next(reader)
+                    reader = csv.reader(f, dialect=DIALECTS.get(detected_format, "excel"))
+                    headings = next(reader, None)
                     if headings:
-                        click.echo("Available fields:")
+                        click.echo(f"Available fields in '{os.path.basename(previous)}' ({len(headings)} total):")
                         for field in headings:
                             click.echo(f"  {field}")
                     else:
                         click.echo("No fields found: CSV file appears to be empty")
         except Exception as e:
+            logger.error(f"Error listing fields: {str(e)}")
             raise click.ClickException(f"Could not list fields: {e}")
         return
 
@@ -310,11 +346,23 @@ def cli(
     start = time.time()
 
     try:
-        # --- Streaming logic ---
-        if (streaming or (detected_format in ("csv", "tsv") and key)) and not fields:
+        # --- Determine comparison method ---
+        use_streaming = False
+        
+        # Check if streaming mode is appropriate
+        if streaming:
             if detected_format not in ("csv", "tsv"):
+                logger.error("Streaming mode requested but file format is not CSV/TSV")
                 raise click.ClickException("--streaming only works with CSV/TSV files.")
-                
+            use_streaming = True
+        elif detected_format in ("csv", "tsv") and not fields and os.path.getsize(previous) > 100*1024*1024:
+            # Auto-enable streaming for large CSV/TSV files (>100MB)
+            logger.info(f"Auto-enabling streaming mode for large file ({os.path.getsize(previous)/1024/1024:.1f} MB)")
+            click.echo("File size is large, automatically enabling streaming mode for better performance...")
+            use_streaming = True
+        
+        # Execute the appropriate comparison method
+        if use_streaming:
             click.echo("Using streaming mode for comparison...")
             diff = streaming_compare_csv(
                 previous,
@@ -345,23 +393,26 @@ def cli(
 
         # Generate output
         if output == "json":
-            print(json.dumps(diff, indent=4))
+            click.echo("Generating JSON output...")
+            json_output = json.dumps(diff, indent=2)
+            print(json_output)
             click.echo("Finished. Output printed as JSON to terminal.")
         elif output == "jsonfile":
+            click.echo(f"Saving JSON output to {output_file}...")
             with open(output_file, "w", encoding=encoding) as f:
-                f.write(json.dumps(diff, indent=4))
+                json.dump(diff, f, indent=2)
             click.echo(f"Finished. Output saved to file: {output_file}")
         else:
-            print(
-                human_text(diff, key)
-            )
+            click.echo("Generating human-readable output...")
+            human_output = human_text(diff, key)
+            print(human_output)
             click.echo("Finished. Output printed in readable format to terminal.")
     
     except Exception as e:
         logger.error(f"Error during comparison: {str(e)}")
         if verbose:
-            import traceback
-            logger.debug(traceback.format_exc())
+            error_traceback = traceback.format_exc()
+            logger.debug(f"Full traceback:\n{error_traceback}")
         raise click.ClickException(str(e))
 
     # Always display the elapsed time
@@ -370,14 +421,23 @@ def cli(
     
     # Display summary statistics
     summary = []
-    for key in ["columns_added", "columns_removed", "added", "removed", "changed"]:
-        if diff[key]:
-            summary.append(f"{len(diff[key])} {key.replace('_', ' ')}")
+    for stat_key in ["columns_added", "columns_removed", "added", "removed", "changed"]:
+        if diff[stat_key]:
+            count = len(diff[stat_key])
+            label = stat_key.replace('_', ' ')
+            summary.append(f"{count} {label}")
     
     if summary:
         click.echo(f"Summary: {', '.join(summary)}")
     else:
         click.echo("No differences found")
+        
+    # Display memory usage if verbose
+    if verbose:
+        import psutil
+        process = psutil.Process(os.getpid())
+        memory_mb = process.memory_info().rss / 1024 / 1024
+        logger.debug(f"Memory usage: {memory_mb:.2f} MB")
 
 if __name__ == "__main__":
     cli()
diff --git a/setup.py b/setup.py
index 02e6b39..c822752 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,8 @@ def get_long_description():
     install_requires=[
         "click>=8.0.0",
         "dictdiffer>=0.9.0",
-        "xxhash>=3.0.0"
+        "xxhash>=3.0.0",
+        "psutil>=5.9.0"
     ],
     entry_points={
         "console_scripts": [

From 2d9eea4d66c3df73a88b7b75e905d7fbee76de83 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 15:12:16 -0500
Subject: [PATCH 100/129] spelling

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 704bdfc..1d323d9 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -1,5 +1,5 @@
 """
-Command-line interface for DB-Diff tool.
+Command-line interface for db-diff tool.
 
 This module provides the command-line interface for comparing CSV and JSON files.
 """

From 44557700cfb8b4f64b670ef2b0501420effc7d2e Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 15:42:50 -0500
Subject: [PATCH 101/129] bug fixes

---
 README.md           |  20 ++++-
 db_diff/__init__.py | 174 ++++++++++++++++++++++++++++++--------------
 db_diff/cli.py      |   4 +-
 3 files changed, 136 insertions(+), 62 deletions(-)

diff --git a/README.md b/README.md
index 469553c..a9d5fd8 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,15 @@
 # db-diff
 
-[![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
+[![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)]- **Key features:**
+
+- **Custom Key Column:** Use `--key` to specify which column uniquely identifies rows.
+- **Output Formats:** Choose between human-readable (`readable`), JSON (`json`), or save JSON to a file (`jsonfile`).
+- **Field Selection:** Use `--fields` to compare only specific columns, or `--ignorefields` to exclude columns.
+- **Streaming Mode:** For very large files, use `--streaming` (CSV/TSV only) to reduce memory usage.
+- **Encoding:** Specify file encoding with `--encoding`.
+- **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
+- **List Fields:** Use `--listfields` to print available columns and exit.
+- **Timing:** Use `--time` to display how long the diff operation took.ithub.com/datsom1/db-diff/releases)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
 
 > **db-diff** is the most advanced and user-friendly Python CLI tool and library for comparing CSV, TSV, and JSON database dumps. Designed specifically for database workflows, it delivers precise, human-readable or machine-readable diffs, supports custom key columns, handles massive files efficiently with streaming mode, and offers flexible field selection.
@@ -248,7 +257,7 @@ A summary of key options:
 | `--showunchanged`     | Show all fields for changed records, not just changed fields                                 |
 | `--format TEXT`       | Explicitly specify input format: `csv`, `tsv`, or `json` (default: auto-detect)              |
 | `--encoding TEXT`     | Input file encoding (default: utf-8)                                                         |
-| `--streaming`         | Use streaming mode for very large CSV/TSV files (requires files to be sorted by key)         |
+| `--streaming`         | Use streaming mode for very large CSV/TSV files to reduce memory usage                       |
 | `--listfields`        | List available fields/columns in the input files and exit                                    |
 | `--verbose`, `-v`     | Enable verbose output with detailed logging                                                  |
 | `--version`           | Show the version and exit                                                                    |
@@ -294,6 +303,8 @@ db-diff one.csv two.csv --key=Id --ignorefields=LastModifiedDate
 db-diff large1.csv large2.csv --key=Id --streaming
 ```
 
+The streaming mode works with CSV/TSV files, reducing memory usage for processing very large files.
+
 **Verbose output with detailed logging:**
 
 ```sh
@@ -304,8 +315,9 @@ db-diff one.csv two.csv --key=Id --verbose
 
 ## Performance
 
-- For very large files, use `--streaming` mode (CSV/TSV only, files must be sorted by key).
-- Supports efficient memory usage and fast comparison for millions of rows.
+- For very large files, use `--streaming` mode for CSV/TSV files to reduce memory usage.
+- Streaming mode reads the files multiple times but keeps less data in memory.
+- The tool supports efficient memory usage and fast comparison for millions of rows.
 
 ---
 
diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index ff4d9d0..d9d1e1b 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -51,7 +51,16 @@ def load_csv(fp, key=None, dialect=None):
             pass
     reader = csv.reader(fp, dialect=(dialect or "excel"))
     headings = next(reader)
-    rows = [dict(zip(headings, line)) for line in reader]
+    
+    # Process each row to ensure consistent handling of values
+    rows = []
+    for line in reader:
+        # Create a dictionary from the header and values
+        row_dict = dict(zip(headings, line))
+        # Process string values to ensure consistency with streaming mode
+        processed_row = {k: v.strip() if isinstance(v, str) else v for k, v in row_dict.items()}
+        rows.append(processed_row)
+    
     if key:
         keyfn = lambda r: r[key]
     else:
@@ -235,7 +244,8 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
 
 def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignorefields=None, encoding='utf-8', dialect='excel'):
     """
-    Efficiently compare two sorted CSV files by streaming, returning a diff dict.
+    Efficiently compare two CSV files by streaming, returning a diff dict.
+    This method uses less memory than loading the entire files into memory.
     
     Args:
         prev_path: Path to the previous CSV file
@@ -258,64 +268,116 @@ def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignor
     }
     
     try:
-        with open(prev_path, newline='', encoding=encoding) as f1, open(curr_path, newline='', encoding=encoding) as f2:
-            reader1 = csv.DictReader(f1, dialect=dialect)
-            reader2 = csv.DictReader(f2, dialect=dialect)
-            
-            if not reader1.fieldnames or not reader2.fieldnames:
-                logger.error("One or both CSV files appear to be empty or malformed")
+        # First pass: Read all keys from both files to determine added and removed rows
+        prev_keys = set()
+        curr_keys = set()
+        prev_columns = set()
+        curr_columns = set()
+        
+        # Ensure consistent handling of CSV values between streaming and non-streaming modes
+        def process_csv_row(row_dict):
+            # Apply the same transformations that would happen in the non-streaming mode
+            # This ensures consistent handling of values between modes
+            return {k: v.strip() if isinstance(v, str) else v for k, v in row_dict.items()}
+        
+        # Load all keys from previous file
+        logger.debug("Streaming: Reading keys from previous file")
+        with open(prev_path, newline='', encoding=encoding) as f:
+            reader = csv.DictReader(f, dialect=dialect)
+            if not reader.fieldnames:
+                logger.error("Previous CSV file appears to be empty or malformed")
                 return result
-                
-            if key not in reader1.fieldnames:
+            if key not in reader.fieldnames:
                 raise ValueError(f"Key column '{key}' not found in previous file")
-            if key not in reader2.fieldnames:
+            
+            prev_columns = set(reader.fieldnames)
+            for row in reader:
+                processed_row = process_csv_row(row)
+                prev_keys.add(processed_row[key])
+        
+        # Load all keys from current file
+        logger.debug("Streaming: Reading keys from current file")
+        with open(curr_path, newline='', encoding=encoding) as f:
+            reader = csv.DictReader(f, dialect=dialect)
+            if not reader.fieldnames:
+                logger.error("Current CSV file appears to be empty or malformed")
+                return result
+            if key not in reader.fieldnames:
                 raise ValueError(f"Key column '{key}' not found in current file")
+            
+            curr_columns = set(reader.fieldnames)
+            for row in reader:
+                processed_row = process_csv_row(row)
+                curr_keys.add(processed_row[key])
+        
+        # Second pass: Process each file to extract data for added, removed and changed rows
+        logger.debug("Streaming: Determining added and removed rows")
+        removed_keys = prev_keys - curr_keys
+        added_keys = curr_keys - prev_keys
+        common_keys = prev_keys & curr_keys
+        
+        # Load previous file data for removed and changed rows
+        prev_data = {}
+        logger.debug("Streaming: Loading data for removed and changed rows from previous file")
+        with open(prev_path, newline='', encoding=encoding) as f:
+            reader = csv.DictReader(f, dialect=dialect)
+            
+            for row in reader:
+                processed_row = process_csv_row(row)
+                key_val = processed_row[key]
+                if key_val in removed_keys or key_val in common_keys:
+                    prev_data[key_val] = processed_row
+        
+        # Load current file data for added and changed rows
+        curr_data = {}
+        logger.debug("Streaming: Loading data for added and changed rows from current file")
+        with open(curr_path, newline='', encoding=encoding) as f:
+            reader = csv.DictReader(f, dialect=dialect)
+            
+            for row in reader:
+                processed_row = process_csv_row(row)
+                key_val = processed_row[key]
+                if key_val in added_keys or key_val in common_keys:
+                    curr_data[key_val] = processed_row
+        
+        # Determine columns to compare
+        compare_columns_set, ignore_columns = _determine_columns_to_compare(
+            prev_columns, curr_columns, compare_columns, ignorefields
+        )
+        
+        # Identify column changes
+        result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns_set]
+        result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns_set]
+        
+        # Add removed rows to result
+        logger.debug(f"Streaming: Processing {len(removed_keys)} removed rows")
+        for key_val in removed_keys:
+            result["removed"].append(prev_data[key_val])
+        
+        # Add added rows to result
+        logger.debug(f"Streaming: Processing {len(added_keys)} added rows")
+        for key_val in added_keys:
+            result["added"].append(curr_data[key_val])
+        
+        # Process changed rows
+        logger.debug(f"Streaming: Processing {len(common_keys)} common rows for changes")
+        for key_val in common_keys:
+            prev_row = prev_data[key_val]
+            curr_row = curr_data[key_val]
+            
+            # Check for changes in compared columns
+            changed_fields = {
+                col: [prev_row.get(col), curr_row.get(col)]
+                for col in compare_columns_set
+                if col in prev_columns and col in curr_columns and prev_row.get(col) != curr_row.get(col)
+            }
+            
+            if changed_fields:
+                result["changed"].append({
+                    "key": key_val,
+                    "changes": changed_fields
+                })
                 
-            prev_row = next(reader1, None)
-            curr_row = next(reader2, None)
-
-            prev_columns = set(reader1.fieldnames)
-            curr_columns = set(reader2.fieldnames)
-
-            # Determine columns to compare
-            compare_columns, _ = _determine_columns_to_compare(prev_columns, curr_columns, compare_columns, ignorefields)
-
-            result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns]
-            result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns]
-
-            while prev_row or curr_row:
-                if prev_row and curr_row:
-                    prev_key_val = prev_row[key]
-                    curr_key_val = curr_row[key]
-                    
-                    if prev_key_val == curr_key_val:
-                        # Check for changes
-                        changed_fields = {
-                            col: [prev_row.get(col), curr_row.get(col)]
-                            for col in compare_columns
-                            if col in prev_columns and col in curr_columns and prev_row.get(col) != curr_row.get(col)
-                        }
-                        if changed_fields:
-                            result["changed"].append({
-                                "key": prev_key_val,
-                                "changes": changed_fields
-                            })
-                        prev_row = next(reader1, None)
-                        curr_row = next(reader2, None)
-                    elif prev_key_val < curr_key_val:
-                        # Row removed
-                        result["removed"].append(prev_row)
-                        prev_row = next(reader1, None)
-                    else:
-                        # Row added
-                        result["added"].append(curr_row)
-                        curr_row = next(reader2, None)
-                elif prev_row:
-                    result["removed"].append(prev_row)
-                    prev_row = next(reader1, None)
-                elif curr_row:
-                    result["added"].append(curr_row)
-                    curr_row = next(reader2, None)
     except (csv.Error, UnicodeDecodeError) as e:
         logger.error(f"CSV processing error: {str(e)}")
         raise
diff --git a/db_diff/cli.py b/db_diff/cli.py
index 1d323d9..30560b0 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -234,7 +234,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
 @click.option(
     "--streaming",
     is_flag=True,
-    help="Use streaming mode for very large CSV/TSV files (requires files to be sorted by key).",
+    help="Use streaming mode for very large CSV/TSV files to reduce memory usage.",
 )
 @click.option(
     "--listfields",
@@ -358,7 +358,7 @@ def cli(
         elif detected_format in ("csv", "tsv") and not fields and os.path.getsize(previous) > 100*1024*1024:
             # Auto-enable streaming for large CSV/TSV files (>100MB)
             logger.info(f"Auto-enabling streaming mode for large file ({os.path.getsize(previous)/1024/1024:.1f} MB)")
-            click.echo("File size is large, automatically enabling streaming mode for better performance...")
+            click.echo("File size is large, automatically enabling streaming mode for better performance and lower memory usage...")
             use_streaming = True
         
         # Execute the appropriate comparison method

From f83c6ef002228beb780cfd5a614039b4e4a4c4a5 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 16:13:51 -0500
Subject: [PATCH 102/129] bug fix

---
 db_diff/cli.py | 74 ++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 66 insertions(+), 8 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 30560b0..4ae6047 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -128,7 +128,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         Full path to the output file
         
     Raises:
-        click.ClickException: If the output path doesn't exist
+        click.ClickException: If the output path can't be created or written to
     """
     if timestamp is None:
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
@@ -147,10 +147,48 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     
     # Handle output path
     if output_path:
-        if not os.path.isdir(output_path):
-            logger.error(f"Output path '{output_path}' does not exist")
-            raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
-        output_file = os.path.join(output_path, os.path.basename(output_file))
+        # Normalize the path (convert backslashes, handle relative paths, etc.)
+        normalized_path = os.path.normpath(os.path.abspath(output_path))
+        logger.debug(f"Normalized output path: {normalized_path}")
+        
+        # Check if directory exists first, to provide better error messages
+        if os.path.exists(normalized_path):
+            if not os.path.isdir(normalized_path):
+                logger.error(f"Output path exists but is not a directory: {normalized_path}")
+                raise click.ClickException(f"Output path '{normalized_path}' exists but is not a directory.")
+            logger.debug(f"Output directory exists: {normalized_path}")
+        else:
+            logger.debug(f"Output directory does not exist, will create: {normalized_path}")
+            try:
+                # Create the directory if it doesn't exist
+                os.makedirs(normalized_path, exist_ok=True)
+                logger.debug(f"Created output directory: {normalized_path}")
+            except OSError as e:
+                logger.error(f"Failed to create output directory '{normalized_path}': {str(e)}")
+                raise click.ClickException(f"Failed to create output directory '{normalized_path}': {e}")
+        
+        # Ensure parent directory of the file exists
+        output_file_path = os.path.join(normalized_path, os.path.basename(output_file))
+        output_dir = os.path.dirname(output_file_path)
+        if not os.path.exists(output_dir):
+            try:
+                os.makedirs(output_dir, exist_ok=True)
+                logger.debug(f"Created parent directory for output file: {output_dir}")
+            except OSError as e:
+                logger.error(f"Failed to create parent directory for output file '{output_dir}': {str(e)}")
+                raise click.ClickException(f"Failed to create parent directory for output file '{output_dir}': {e}")
+                
+        # Test write permissions by attempting to open the file
+        try:
+            with open(output_file_path, 'w') as _:
+                pass
+            os.remove(output_file_path)  # Remove the test file
+            logger.debug(f"Verified write permissions for output file: {output_file_path}")
+        except IOError as e:
+            logger.error(f"Cannot write to output file '{output_file_path}': {str(e)}")
+            raise click.ClickException(f"Cannot write to output file '{output_file_path}': {e}")
+            
+        return output_file_path
     
     logger.debug(f"Generated output filename: {output_file}")
     return output_file
@@ -275,6 +313,15 @@ def cli(
     if verbose:
         logger.setLevel(logging.DEBUG)
         click.echo("Verbose mode enabled")
+        
+        # Print diagnostic information about paths
+        logger.debug(f"Current working directory: {os.getcwd()}")
+        if output == "jsonfile" and output_path:
+            logger.debug(f"Output path provided: {output_path}")
+            logger.debug(f"Normalized output path: {os.path.normpath(os.path.abspath(output_path))}")
+            logger.debug(f"Path exists: {os.path.exists(output_path)}")
+            logger.debug(f"Path is directory: {os.path.isdir(output_path)}")
+            logger.debug(f"Path is absolute: {os.path.isabs(output_path)}")
     else:
         logger.setLevel(logging.INFO)
     
@@ -399,9 +446,20 @@ def cli(
             click.echo("Finished. Output printed as JSON to terminal.")
         elif output == "jsonfile":
             click.echo(f"Saving JSON output to {output_file}...")
-            with open(output_file, "w", encoding=encoding) as f:
-                json.dump(diff, f, indent=2)
-            click.echo(f"Finished. Output saved to file: {output_file}")
+            try:
+                # Ensure the parent directory exists
+                output_dir = os.path.dirname(output_file)
+                if output_dir and not os.path.exists(output_dir):
+                    logger.debug(f"Creating parent directory: {output_dir}")
+                    os.makedirs(output_dir, exist_ok=True)
+                
+                # Write the file
+                with open(output_file, "w", encoding=encoding) as f:
+                    json.dump(diff, f, indent=2)
+                click.echo(f"Finished. Output saved to file: {output_file}")
+            except IOError as e:
+                logger.error(f"Failed to write to file '{output_file}': {str(e)}")
+                raise click.ClickException(f"Failed to write to output file '{output_file}': {e}")
         else:
             click.echo("Generating human-readable output...")
             human_output = human_text(diff, key)

From 2c9eb551d993db0205ab0db953ae4e3dd3fdaa3e Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Mon, 30 Jun 2025 16:14:37 -0500
Subject: [PATCH 103/129] bug fix

---
 db_diff/cli.py | 74 ++++++--------------------------------------------
 1 file changed, 8 insertions(+), 66 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 4ae6047..30560b0 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -128,7 +128,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         Full path to the output file
         
     Raises:
-        click.ClickException: If the output path can't be created or written to
+        click.ClickException: If the output path doesn't exist
     """
     if timestamp is None:
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
@@ -147,48 +147,10 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     
     # Handle output path
     if output_path:
-        # Normalize the path (convert backslashes, handle relative paths, etc.)
-        normalized_path = os.path.normpath(os.path.abspath(output_path))
-        logger.debug(f"Normalized output path: {normalized_path}")
-        
-        # Check if directory exists first, to provide better error messages
-        if os.path.exists(normalized_path):
-            if not os.path.isdir(normalized_path):
-                logger.error(f"Output path exists but is not a directory: {normalized_path}")
-                raise click.ClickException(f"Output path '{normalized_path}' exists but is not a directory.")
-            logger.debug(f"Output directory exists: {normalized_path}")
-        else:
-            logger.debug(f"Output directory does not exist, will create: {normalized_path}")
-            try:
-                # Create the directory if it doesn't exist
-                os.makedirs(normalized_path, exist_ok=True)
-                logger.debug(f"Created output directory: {normalized_path}")
-            except OSError as e:
-                logger.error(f"Failed to create output directory '{normalized_path}': {str(e)}")
-                raise click.ClickException(f"Failed to create output directory '{normalized_path}': {e}")
-        
-        # Ensure parent directory of the file exists
-        output_file_path = os.path.join(normalized_path, os.path.basename(output_file))
-        output_dir = os.path.dirname(output_file_path)
-        if not os.path.exists(output_dir):
-            try:
-                os.makedirs(output_dir, exist_ok=True)
-                logger.debug(f"Created parent directory for output file: {output_dir}")
-            except OSError as e:
-                logger.error(f"Failed to create parent directory for output file '{output_dir}': {str(e)}")
-                raise click.ClickException(f"Failed to create parent directory for output file '{output_dir}': {e}")
-                
-        # Test write permissions by attempting to open the file
-        try:
-            with open(output_file_path, 'w') as _:
-                pass
-            os.remove(output_file_path)  # Remove the test file
-            logger.debug(f"Verified write permissions for output file: {output_file_path}")
-        except IOError as e:
-            logger.error(f"Cannot write to output file '{output_file_path}': {str(e)}")
-            raise click.ClickException(f"Cannot write to output file '{output_file_path}': {e}")
-            
-        return output_file_path
+        if not os.path.isdir(output_path):
+            logger.error(f"Output path '{output_path}' does not exist")
+            raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
+        output_file = os.path.join(output_path, os.path.basename(output_file))
     
     logger.debug(f"Generated output filename: {output_file}")
     return output_file
@@ -313,15 +275,6 @@ def cli(
     if verbose:
         logger.setLevel(logging.DEBUG)
         click.echo("Verbose mode enabled")
-        
-        # Print diagnostic information about paths
-        logger.debug(f"Current working directory: {os.getcwd()}")
-        if output == "jsonfile" and output_path:
-            logger.debug(f"Output path provided: {output_path}")
-            logger.debug(f"Normalized output path: {os.path.normpath(os.path.abspath(output_path))}")
-            logger.debug(f"Path exists: {os.path.exists(output_path)}")
-            logger.debug(f"Path is directory: {os.path.isdir(output_path)}")
-            logger.debug(f"Path is absolute: {os.path.isabs(output_path)}")
     else:
         logger.setLevel(logging.INFO)
     
@@ -446,20 +399,9 @@ def cli(
             click.echo("Finished. Output printed as JSON to terminal.")
         elif output == "jsonfile":
             click.echo(f"Saving JSON output to {output_file}...")
-            try:
-                # Ensure the parent directory exists
-                output_dir = os.path.dirname(output_file)
-                if output_dir and not os.path.exists(output_dir):
-                    logger.debug(f"Creating parent directory: {output_dir}")
-                    os.makedirs(output_dir, exist_ok=True)
-                
-                # Write the file
-                with open(output_file, "w", encoding=encoding) as f:
-                    json.dump(diff, f, indent=2)
-                click.echo(f"Finished. Output saved to file: {output_file}")
-            except IOError as e:
-                logger.error(f"Failed to write to file '{output_file}': {str(e)}")
-                raise click.ClickException(f"Failed to write to output file '{output_file}': {e}")
+            with open(output_file, "w", encoding=encoding) as f:
+                json.dump(diff, f, indent=2)
+            click.echo(f"Finished. Output saved to file: {output_file}")
         else:
             click.echo("Generating human-readable output...")
             human_output = human_text(diff, key)

From 3f949a7c5717bd072ef0097ca50771ac2d01f3ae Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 11:33:27 -0500
Subject: [PATCH 104/129] Added different streaming modes. [auto, sortedkey,
 normal, off]

---
 README.md           |  60 ++++++--
 db_diff/__init__.py | 368 ++++++++++++++++++++++++++++++++++----------
 db_diff/cli.py      |  66 ++++++--
 3 files changed, 391 insertions(+), 103 deletions(-)

diff --git a/README.md b/README.md
index a9d5fd8..f234899 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 - **Custom Key Column:** Use `--key` to specify which column uniquely identifies rows.
 - **Output Formats:** Choose between human-readable (`readable`), JSON (`json`), or save JSON to a file (`jsonfile`).
 - **Field Selection:** Use `--fields` to compare only specific columns, or `--ignorefields` to exclude columns.
-- **Streaming Mode:** For very large files, use `--streaming` (CSV/TSV only) to reduce memory usage.
+- **Streaming Mode Options:** For very large files, choose from multiple streaming modes (`--streaming=auto|normal|sortedkey|off`) to optimize performance.
 - **Encoding:** Specify file encoding with `--encoding`.
 - **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
 - **List Fields:** Use `--listfields` to print available columns and exit.
@@ -130,7 +130,11 @@ db-diff [OPTIONS] PREVIOUS CURRENT
 - **Custom Key Column:** Use `--key` to specify which column uniquely identifies rows.
 - **Output Formats:** Choose between human-readable (`readable`), JSON (`json`), or save JSON to a file (`jsonfile`).
 - **Field Selection:** Use `--fields` to compare only specific columns, or `--ignorefields` to exclude columns.
-- **Streaming Mode:** For very large files, use `--streaming` (CSV/TSV only, files must be sorted by key).
+- **Streaming Mode Options:** Choose between different streaming modes for CSV/TSV files:
+  - `--streaming=auto` (default): Automatically select the best mode based on file size and key ordering
+  - `--streaming=normal`: Memory-efficient but slower processing
+  - `--streaming=sortedkey`: Very fast streaming mode for files with sorted key columns
+  - `--streaming=off`: Disable streaming regardless of file size
 - **Encoding:** Specify file encoding with `--encoding`.
 - **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
 - **List Fields:** Use `--listfields` to print available columns and exit.
@@ -178,6 +182,7 @@ diff = compare(prev, curr, show_unchanged=False)
 #### Streaming Comparison (for large CSV/TSV files)
 
 ```python
+# For normal streaming mode (memory efficient)
 from db_diff import streaming_compare_csv
 
 diff = streaming_compare_csv(
@@ -188,6 +193,28 @@ diff = streaming_compare_csv(
     encoding="utf-8",
     dialect="excel"
 )
+
+# For sorted key streaming mode (faster, but requires sorted keys)
+from db_diff import streaming_compare_csv_sorted
+
+diff = streaming_compare_csv_sorted(
+    "one.csv",
+    "two.csv",
+    key="Id",
+    compare_columns={"Id", "name", "age"},
+    encoding="utf-8",
+    dialect="excel"
+)
+
+# Check if a file's key column is sorted
+from db_diff import is_key_column_sorted
+
+is_sorted = is_key_column_sorted(
+    "one.csv",
+    key="Id",
+    encoding="utf-8",
+    dialect="excel"
+)
 ```
 
 #### Diff Result Structure
@@ -257,7 +284,7 @@ A summary of key options:
 | `--showunchanged`     | Show all fields for changed records, not just changed fields                                 |
 | `--format TEXT`       | Explicitly specify input format: `csv`, `tsv`, or `json` (default: auto-detect)              |
 | `--encoding TEXT`     | Input file encoding (default: utf-8)                                                         |
-| `--streaming`         | Use streaming mode for very large CSV/TSV files to reduce memory usage                       |
+| `--streaming=MODE`    | Streaming mode for CSV/TSV files. Options: `auto` (default), `normal`, `sortedkey`, `off`    |
 | `--listfields`        | List available fields/columns in the input files and exit                                    |
 | `--verbose`, `-v`     | Enable verbose output with detailed logging                                                  |
 | `--version`           | Show the version and exit                                                                    |
@@ -297,13 +324,23 @@ db-diff one.csv two.csv --key=Id --fields=Id,name
 db-diff one.csv two.csv --key=Id --ignorefields=LastModifiedDate
 ```
 
-**Streaming mode for large files:**
+**Streaming mode options for large files:**
 
 ```sh
-db-diff large1.csv large2.csv --key=Id --streaming
+# Auto mode (default) - automatically selects the best streaming mode
+db-diff large1.csv large2.csv --key=Id --streaming=auto
+
+# Normal streaming mode - memory efficient but slower
+db-diff large1.csv large2.csv --key=Id --streaming=normal
+
+# Sorted key streaming mode - very fast for files with sorted keys
+db-diff large1.csv large2.csv --key=Id --streaming=sortedkey
+
+# Disable streaming regardless of file size
+db-diff large1.csv large2.csv --key=Id --streaming=off
 ```
 
-The streaming mode works with CSV/TSV files, reducing memory usage for processing very large files.
+The streaming modes work with CSV/TSV files only. The `auto` mode will automatically detect if the key columns are sorted and select the most efficient streaming mode for your data.
 
 **Verbose output with detailed logging:**
 
@@ -315,9 +352,14 @@ db-diff one.csv two.csv --key=Id --verbose
 
 ## Performance
 
-- For very large files, use `--streaming` mode for CSV/TSV files to reduce memory usage.
-- Streaming mode reads the files multiple times but keeps less data in memory.
-- The tool supports efficient memory usage and fast comparison for millions of rows.
+- Choose the appropriate streaming mode for your data:
+  - `auto` (default): Automatically selects the best mode based on file properties
+  - `sortedkey`: Fastest streaming mode, but requires key columns to be sorted in both files
+  - `normal`: Memory-efficient but slower, good for very large unsorted files
+  - `off`: Use when you want to process the entire files in memory regardless of size
+- The tool automatically detects if key columns are sorted and selects the most efficient mode
+- For very large files, the sorted key streaming mode can be 2-5x faster than normal streaming
+- Streaming modes read files in chunks, so they use significantly less memory than loading entire files
 
 ---
 
diff --git a/db_diff/__init__.py b/db_diff/__init__.py
index d9d1e1b..1919fa0 100644
--- a/db_diff/__init__.py
+++ b/db_diff/__init__.py
@@ -384,99 +384,307 @@ def process_csv_row(row_dict):
         
     return result
 
-def human_text(result, key=None):
+def streaming_compare_csv_sorted(prev_path, curr_path, key, compare_columns=None, ignorefields=None, encoding='utf-8', dialect='excel'):
     """
-    Render the diff result as a human-readable string.
+    Efficiently compare two CSV files by streaming with sorted keys, returning a diff dict.
+    This method uses less memory than loading the entire files into memory and is faster
+    than the standard streaming mode because it assumes keys are sorted in both files.
     
     Args:
-        result: Diff result dictionary
-        key: Name of the key column (optional)
+        prev_path: Path to the previous CSV file
+        curr_path: Path to the current CSV file
+        key: Column name to use as key for comparison
+        compare_columns: List of columns to compare (optional)
+        ignorefields: List of columns to ignore (optional)
+        encoding: File encoding (default: utf-8)
+        dialect: CSV dialect (default: excel)
         
     Returns:
-        Formatted string with diff information
+        Dictionary containing diff information
     """
-    title = []
-    summary = []
-    show_headers = sum(1 for k in result if result[k]) > 1
+    result = {
+        "added": [],
+        "removed": [],
+        "changed": [],
+        "columns_added": [],
+        "columns_removed": [],
+    }
     
-    # Handle column changes
-    if result["columns_added"]:
-        fragment = f"{len(result['columns_added'])} {'column' if len(result['columns_added']) == 1 else 'columns'} added"
-        title.append(fragment)
-        summary.extend(
-            [fragment, ""]
-            + [f"  {c}" for c in sorted(result["columns_added"])]
-            + [""]
-        )
-    if result["columns_removed"]:
-        fragment = f"{len(result['columns_removed'])} {'column' if len(result['columns_removed']) == 1 else 'columns'} removed"
-        title.append(fragment)
-        summary.extend(
-            [fragment, ""]
-            + [f"  {c}" for c in sorted(result["columns_removed"])]
-            + [""]
+    try:
+        # Extract column information from both files
+        logger.debug("Streaming sorted: Extracting column information")
+        prev_columns = set()
+        curr_columns = set()
+        
+        with open(prev_path, newline='', encoding=encoding) as f:
+            reader = csv.reader(f, dialect=dialect)
+            prev_headings = next(reader, None)
+            if not prev_headings:
+                logger.error("Previous file has no headers")
+                raise ValueError("Previous file has no headers")
+            prev_columns = set(prev_headings)
+            if key not in prev_headings:
+                logger.error(f"Key column '{key}' not found in previous file")
+                raise ValueError(f"Key column '{key}' not found in previous file")
+            prev_key_index = prev_headings.index(key)
+        
+        with open(curr_path, newline='', encoding=encoding) as f:
+            reader = csv.reader(f, dialect=dialect)
+            curr_headings = next(reader, None)
+            if not curr_headings:
+                logger.error("Current file has no headers")
+                raise ValueError("Current file has no headers")
+            curr_columns = set(curr_headings)
+            if key not in curr_headings:
+                logger.error(f"Key column '{key}' not found in current file")
+                raise ValueError(f"Key column '{key}' not found in current file")
+            curr_key_index = curr_headings.index(key)
+            
+        # Determine columns to compare
+        compare_columns_set, ignore_columns = _determine_columns_to_compare(
+            prev_columns, curr_columns, compare_columns, ignorefields
         )
         
-    # Handle changed rows
-    if result["changed"]:
-        fragment = f"{len(result['changed'])} rows changed"
-        title.append(fragment)
-        if show_headers:
-            summary.append(fragment + "\n")
-        change_blocks = []
-        for details in result["changed"]:
-            block = []
-            block.append(f"  {key}: {details['key']}")
-            for field, (prev_value, current_value) in details["changes"].items():
-                block.append(f'    {field}: "{prev_value}" => "{current_value}"')
-            block.append("")
-            change_blocks.append("\n".join(block))
-            if details.get("unchanged"):
-                block = []
-                block.append("    Unchanged:")
-                for field, value in details["unchanged"].items():
-                    block.append(f'      {field}: "{value}"')
-                block.append("")
-                change_blocks.append("\n".join(block))
-        summary.append("\n".join(change_blocks))
-        
-    # Handle added rows
-    if result["added"]:
-        fragment = f"{len(result['added'])} rows added"
-        title.append(fragment)
-        if show_headers:
-            summary.append(fragment + "\n")
-        rows = []
-        for row in result["added"]:
-            to_append = human_row(row, prefix="  ")
-            rows.append(to_append)
-        summary.append("\n\n".join(rows))
-        summary.append("")
-        
-    # Handle removed rows
-    if result["removed"]:
-        fragment = f"{len(result['removed'])} rows removed"
-        title.append(fragment)
-        if show_headers:
-            summary.append(fragment + "\n")
-        rows = []
-        for row in result["removed"]:
-            to_append = human_row(row, prefix="  ")
-            rows.append(to_append)
-        summary.append("\n\n".join(rows))
-        summary.append("")
-        
-    return (", ".join(title) + "\n\n" + ("\n".join(summary))).strip()
+        # Identify column changes
+        result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns_set]
+        result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns_set]
+        
+        # Open both files and compare row by row
+        logger.debug("Streaming sorted: Starting row comparison")
+        with open(prev_path, newline='', encoding=encoding) as prev_file, \
+             open(curr_path, newline='', encoding=encoding) as curr_file:
+            
+            # Skip headers
+            prev_reader = csv.reader(prev_file, dialect=dialect)
+            curr_reader = csv.reader(curr_file, dialect=dialect)
+            prev_headers = next(prev_reader)
+            curr_headers = next(curr_reader)
+            
+            # Set up dictionaries to convert row arrays to dicts
+            prev_indices = {name: i for i, name in enumerate(prev_headers)}
+            curr_indices = {name: i for i, name in enumerate(curr_headers)}
+            
+            # Get initial rows
+            try:
+                prev_row_data = next(prev_reader, None)
+                curr_row_data = next(curr_reader, None)
+            except csv.Error as e:
+                logger.error(f"CSV error: {str(e)}")
+                raise
+            
+            # Process row by row in sorted order
+            while prev_row_data is not None or curr_row_data is not None:
+                # Convert row arrays to dicts
+                prev_row = {name: prev_row_data[i].strip() if prev_row_data else None 
+                           for name, i in prev_indices.items()} if prev_row_data else None
+                curr_row = {name: curr_row_data[i].strip() if curr_row_data else None 
+                           for name, i in curr_indices.items()} if curr_row_data else None
+                
+                # Compare keys
+                prev_key_val = prev_row[key] if prev_row else None
+                curr_key_val = curr_row[key] if curr_row else None
+                
+                if prev_row is None:
+                    # Current row only - add it to added list
+                    result["added"].append(curr_row)
+                    curr_row_data = next(curr_reader, None)
+                elif curr_row is None:
+                    # Previous row only - add it to removed list
+                    result["removed"].append(prev_row)
+                    prev_row_data = next(prev_reader, None)
+                elif prev_key_val < curr_key_val:
+                    # Previous row is removed
+                    result["removed"].append(prev_row)
+                    prev_row_data = next(prev_reader, None)
+                elif prev_key_val > curr_key_val:
+                    # Current row is added
+                    result["added"].append(curr_row)
+                    curr_row_data = next(curr_reader, None)
+                else:
+                    # Same key - check for changes
+                    changed_fields = {
+                        col: [prev_row.get(col), curr_row.get(col)]
+                        for col in compare_columns_set
+                        if col in prev_columns and col in curr_columns and prev_row.get(col) != curr_row.get(col)
+                    }
+                    
+                    if changed_fields:
+                        result["changed"].append({
+                            "key": prev_key_val,
+                            "changes": changed_fields
+                        })
+                    
+                    # Move to next rows
+                    prev_row_data = next(prev_reader, None)
+                    curr_row_data = next(curr_reader, None)
+    
+    except (csv.Error, UnicodeDecodeError) as e:
+        logger.error(f"CSV processing error: {str(e)}")
+        raise
+        
+    return result
 
-def human_row(row, prefix=""):
+def is_key_column_sorted(file_path, key, encoding='utf-8', dialect='excel', sample_size=1000):
     """
-    Render a row as a human-readable string.
+    Check if the key column in a CSV file is sorted in ascending order.
     
     Args:
-        row: Row data dictionary
-        prefix: String prefix for each line
+        file_path: Path to the CSV file
+        key: Column name to check for sorting
+        encoding: File encoding (default: utf-8)
+        dialect: CSV dialect (default: excel)
+        sample_size: Number of rows to check (default: 1000)
+        
+    Returns:
+        Boolean indicating whether the key column appears to be sorted
+    """
+    try:
+        with open(file_path, newline='', encoding=encoding) as f:
+            reader = csv.reader(f, dialect=dialect)
+            headers = next(reader, None)
+            if not headers:
+                return False
+                
+            if key not in headers:
+                return False
+                
+            key_index = headers.index(key)
+            
+            # Read the first row
+            prev_row = next(reader, None)
+            if not prev_row:
+                return True  # Only header row exists, technically sorted
+                
+            prev_key_value = prev_row[key_index].strip()
+            count = 0
+            
+            # Check subsequent rows to see if they're sorted
+            for row in reader:
+                curr_key_value = row[key_index].strip()
+                
+                # If current key is less than previous, not sorted
+                if curr_key_value < prev_key_value:
+                    return False
+                    
+                prev_key_value = curr_key_value
+                count += 1
+                
+                # Check only a sample to avoid reading entire file
+                if count >= sample_size:
+                    break
+                    
+            return True
+            
+    except Exception as e:
+        logger.warning(f"Error checking if key column is sorted: {str(e)}")
+        return False
+
+def human_text(diff, key=None, current=None):
+    """
+    Convert a diff dictionary to human-readable text.
+    
+    Args:
+        diff: Dictionary containing diff information
+        key: Key column name for reference in output (optional)
+        current: Dictionary of current data (optional, used for unchanged fields)
         
     Returns:
-        Formatted string representation of the row
+        Human-readable string representation of the diff
     """
-    return "\n".join(f"{prefix}{key}: {value}" for key, value in row.items())
+    lines = []
+    
+    # Summary counts
+    summary = []
+    if diff["columns_added"]:
+        summary.append(f"{len(diff['columns_added'])} columns added")
+    if diff["columns_removed"]:
+        summary.append(f"{len(diff['columns_removed'])} columns removed")
+    if diff["changed"]:
+        summary.append(f"{len(diff['changed'])} rows changed")
+    if diff["added"]:
+        summary.append(f"{len(diff['added'])} rows added")
+    if diff["removed"]:
+        summary.append(f"{len(diff['removed'])} rows removed")
+        
+    if summary:
+        lines.append(", ".join(summary))
+    else:
+        lines.append("No differences found")
+    
+    lines.append("")  # Blank line after summary
+    
+    # Column changes
+    if diff["columns_added"]:
+        lines.append(f"{len(diff['columns_added'])} columns added")
+        lines.append("")
+        for col in sorted(diff["columns_added"]):
+            lines.append(f"  {col}")
+        lines.append("")
+    
+    if diff["columns_removed"]:
+        lines.append(f"{len(diff['columns_removed'])} columns removed")
+        lines.append("")
+        for col in sorted(diff["columns_removed"]):
+            lines.append(f"  {col}")
+        lines.append("")
+    
+    # Changed rows
+    if diff["changed"]:
+        lines.append(f"{len(diff['changed'])} rows changed")
+        lines.append("")
+        
+        for change in diff["changed"]:
+            change_key = change["key"]
+            lines.append(f"  {key}: {change_key}")
+            
+            # Show changed fields
+            for field, (old, new) in sorted(change["changes"].items()):
+                # Format values for display
+                old_str = f'"{old}"' if isinstance(old, str) else str(old)
+                new_str = f'"{new}"' if isinstance(new, str) else str(new)
+                lines.append(f"    {field}: {old_str} => {new_str}")
+            
+            # Show unchanged fields if present
+            if "unchanged" in change and change["unchanged"]:
+                lines.append("    (unchanged fields)")
+                for field, value in sorted(change["unchanged"].items()):
+                    value_str = f'"{value}"' if isinstance(value, str) else str(value)
+                    lines.append(f"    {field}: {value_str}")
+            
+            lines.append("")  # Blank line between changes
+    
+    # Added rows
+    if diff["added"]:
+        lines.append(f"{len(diff['added'])} rows added")
+        lines.append("")
+        
+        for row in diff["added"]:
+            key_val = row.get(key, "unknown") if key else "unknown"
+            lines.append(f"  {key}: {key_val}")
+            
+            # Show all fields except the key
+            for field, value in sorted(row.items()):
+                if field != key:
+                    value_str = f'"{value}"' if isinstance(value, str) else str(value)
+                    lines.append(f"  {field}: {value_str}")
+            
+            lines.append("")  # Blank line between rows
+    
+    # Removed rows
+    if diff["removed"]:
+        lines.append(f"{len(diff['removed'])} rows removed")
+        lines.append("")
+        
+        for row in diff["removed"]:
+            key_val = row.get(key, "unknown") if key else "unknown"
+            lines.append(f"  {key}: {key_val}")
+            
+            # Show all fields except the key
+            for field, value in sorted(row.items()):
+                if field != key:
+                    value_str = f'"{value}"' if isinstance(value, str) else str(value)
+                    lines.append(f"  {field}: {value_str}")
+            
+            lines.append("")  # Blank line between rows
+    
+    return "\n".join(lines)
diff --git a/db_diff/cli.py b/db_diff/cli.py
index 30560b0..bd5a6bc 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -21,7 +21,9 @@
     load_json, 
     compare, 
     human_text, 
-    streaming_compare_csv, 
+    streaming_compare_csv,
+    streaming_compare_csv_sorted,
+    is_key_column_sorted,
     logger
 )
 
@@ -233,8 +235,9 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
 )
 @click.option(
     "--streaming",
-    is_flag=True,
-    help="Use streaming mode for very large CSV/TSV files to reduce memory usage.",
+    type=click.Choice(["off", "normal", "sortedkey", "auto"], case_sensitive=False),
+    default="auto",
+    help="Streaming mode for processing large CSV/TSV files. Options: off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (default: automatically select best mode).",
 )
 @click.option(
     "--listfields",
@@ -347,23 +350,58 @@ def cli(
 
     try:
         # --- Determine comparison method ---
-        use_streaming = False
+        use_streaming = "normal"  # Default streaming mode if enabled
+        use_streaming_sorted = False  # Whether to use the sorted key streaming mode
         
         # Check if streaming mode is appropriate
-        if streaming:
+        if streaming == "off":
+            logger.debug("Streaming mode manually disabled")
+            use_streaming = None
+        elif streaming in ("normal", "sortedkey"):
             if detected_format not in ("csv", "tsv"):
                 logger.error("Streaming mode requested but file format is not CSV/TSV")
-                raise click.ClickException("--streaming only works with CSV/TSV files.")
-            use_streaming = True
-        elif detected_format in ("csv", "tsv") and not fields and os.path.getsize(previous) > 100*1024*1024:
-            # Auto-enable streaming for large CSV/TSV files (>100MB)
-            logger.info(f"Auto-enabling streaming mode for large file ({os.path.getsize(previous)/1024/1024:.1f} MB)")
-            click.echo("File size is large, automatically enabling streaming mode for better performance and lower memory usage...")
-            use_streaming = True
+                raise click.ClickException("Streaming modes only work with CSV/TSV files.")
+            if streaming == "sortedkey":
+                logger.debug("Using sorted key streaming mode")
+                use_streaming = "sortedkey"
+        elif streaming == "auto":
+            # Auto mode - determine the best streaming mode based on file properties
+            if detected_format not in ("csv", "tsv"):
+                logger.debug("Auto streaming mode: Not CSV/TSV format, disabling streaming")
+                use_streaming = None
+            elif fields:  # If specific fields are selected, we need to read everything, so streaming won't help much
+                logger.debug("Auto streaming mode: Field filtering active, disabling streaming")
+                use_streaming = None
+            elif os.path.getsize(previous) <= 100*1024*1024:  # Less than 10MB
+                logger.debug("Auto streaming mode: Small file size, disabling streaming")
+                use_streaming = None
+            else:
+                # Check if key column is sorted in both files
+                logger.debug("Auto streaming mode: Checking if key columns are sorted")
+                if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel")) and
+                    is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"))):
+                    logger.info("Auto streaming mode: Key columns appear to be sorted, using sorted key streaming mode")
+                    use_streaming = "sortedkey"
+                    click.echo("Key columns appear to be sorted, using fast streaming mode...")
+                elif os.path.getsize(previous) > 100*1024*1024:  # >100MB
+                    logger.info(f"Auto streaming mode: Large file size ({os.path.getsize(previous)/1024/1024:.1f} MB), using normal streaming mode")
+                    use_streaming = "normal"
+                    click.echo("File size is large, automatically enabling streaming mode for better performance and lower memory usage...")
         
         # Execute the appropriate comparison method
-        if use_streaming:
-            click.echo("Using streaming mode for comparison...")
+        if use_streaming == "sortedkey":
+            click.echo("Using sorted key streaming mode for comparison...")
+            diff = streaming_compare_csv_sorted(
+                previous,
+                current,
+                key=key,
+                compare_columns=fields_set,
+                ignorefields=ignorefields_set,
+                encoding=encoding,
+                dialect=DIALECTS.get(detected_format, "excel"),
+            )
+        elif use_streaming == "normal":
+            click.echo("Using normal streaming mode for comparison...")
             diff = streaming_compare_csv(
                 previous,
                 current,

From cfe582dad68f75eb1f4ccde9c020d97e3f403049 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 11:38:47 -0500
Subject: [PATCH 105/129] updated helptext

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index bd5a6bc..d3e9a36 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -237,7 +237,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     "--streaming",
     type=click.Choice(["off", "normal", "sortedkey", "auto"], case_sensitive=False),
     default="auto",
-    help="Streaming mode for processing large CSV/TSV files. Options: off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (default: automatically select best mode).",
+    help="Streaming mode for processing large CSV/TSV files. Options: off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto [default: automatically select best mode]",
 )
 @click.option(
     "--listfields",

From da083f8fc7431815425f52eb228cbc2adf692754 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 11:53:44 -0500
Subject: [PATCH 106/129] made helptext more concise

---
 db_diff/cli.py | 70 ++++++++++++++++++++++++++------------------------
 1 file changed, 37 insertions(+), 33 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index d3e9a36..a434031 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -160,10 +160,32 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
 @click.command(
     context_settings={
         "help_option_names": ["-h", "--help"],
-        "max_content_width": TERMINAL_WIDTH  # Set the max width to terminal width
+        "max_content_width": TERMINAL_WIDTH
     },
-    help="Compare the differences between two CSV or JSON files to find differences.",
-    epilog="Example: db-diff old.csv new.csv --key=Id --output=jsonfile --outfilename=diff.json",
+    help="Compare two CSV or JSON files and identify differences.",
+    epilog="""
+Examples:
+  Basic comparison:
+    db-diff old.csv new.csv
+    
+  With specific key column:
+    db-diff old.csv new.csv --key=Id
+    
+  Output to JSON file:
+    db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json
+    
+  Compare only specific fields:
+    db-diff old.csv new.csv --fields=Id,Name,Email
+    
+  Ignore specific fields:
+    db-diff old.csv new.csv --ignorefields=LastModified,CreatedDate
+    
+  Use streaming mode for large files:
+    db-diff old.csv new.csv --streaming=normal
+    
+  List available fields:
+    db-diff old.csv new.csv --listfields
+    """,
     no_args_is_help=True,
 )
 @click.argument(
@@ -178,71 +200,71 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     "--key",
     type=str,
     default=None,
-    help="Column to use as a unique ID for each row (ex: --key=Id). [default: first column header]",
+    help="Column to use as a unique ID for each row. [default: first column header]",
 )
 @click.option(
     "--output",
     default="readable",
     show_default=True,
     type=click.Choice(["readable", "json", "jsonfile"], case_sensitive=False),
-    help="Output format. Available: (readable|json|jsonfile).",
+    help="Output format (readable|json|jsonfile).",
 )
 @click.option(
     "--outfilename",
     "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
-    help="File to write JSON output to (only used with --output=jsonfile). [default: YYYY-MM-DD_HH-MM-SS_diffs_file1name__VS__file2name.json]",
+    help="File to write JSON output to. [default: YYYY-MM-DD_HH-MM-SS_diffs_file1name__VS__file2name.json]",
 )
 @click.option(
     "--outfilepath",
     "output_path",
     type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
     default=None,
-    help="Directory to save the output file (only used with --output=jsonfile). [default: current directory]",
+    help="Directory to save the output file. [default: current directory]",
 )
 @click.option(
     "--fields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to compare (all others ignored). Mutually exclusive with --ignorefields. (example: --fields=Id,Name,Age,Email).",
+    help="Comma-separated list of fields to compare (all others ignored). Cannot use with --ignorefields. Example: --fields=Id,Name,Email",
 )
 @click.option(
     "--ignorefields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to ignore. Mutually exclusive with --fields. (example: --ignorefields=LastModifiedDate,SystemModstamp).",
+    help="Comma-separated list of fields to ignore. Cannot use with --fields. Example: --ignorefields=LastModifiedDate,SystemModstamp",
 )
 @click.option(
     "--showunchanged",
     "show_unchanged",
     is_flag=True,
-    help="If a record is changed, show ALL fields, not just the changed fields.",
+    help="Show ALL fields for changed records, not just the modified fields.",
 )
 @click.option(
     "--format",
     "input_format",
     default=None,
     type=click.Choice(list(SUPPORTED_FORMATS), case_sensitive=False),
-    help="Explicitly specify input format. Available (csv|tsv|json). [default: auto-detect based on file extension]",
+    help="Input format (csv|tsv|json). [default: auto-detect from file extension]",
 )
 @click.option(
     "--encoding",
     type=str,
     default="utf-8",
     show_default=True,
-    help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).",
+    help="Input file encoding (utf-8|utf-16|latin1|cp1252|ascii|...).",
 )
 @click.option(
     "--streaming",
     type=click.Choice(["off", "normal", "sortedkey", "auto"], case_sensitive=False),
     default="auto",
-    help="Streaming mode for processing large CSV/TSV files. Options: off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto [default: automatically select best mode]",
+    help="Processing mode for large CSV/TSV files: off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (select best mode).",
 )
 @click.option(
     "--listfields",
     is_flag=True,
-    help="List available fields/columns in the input files and exit.",
+    help="List available fields in input files and exit.",
 )
 @click.option(
     "--verbose", "-v",
@@ -255,25 +277,7 @@ def cli(
     show_unchanged, encoding, output, output_file, output_path,
     fields, ignorefields, streaming, listfields, verbose
 ):
-    """
-    Compare the differences between two CSV or JSON files.
-    
-    Args:
-        previous: Path to the previous file
-        current: Path to the current file
-        key: Column name to use as key (optional)
-        input_format: Format of the input files (optional)
-        show_unchanged: Whether to show unchanged fields
-        encoding: File encoding
-        output: Output format
-        output_file: Output filename (optional)
-        output_path: Output directory (optional)
-        fields: List of fields to compare (optional)
-        ignorefields: List of fields to ignore (optional)
-        streaming: Whether to use streaming mode
-        listfields: Whether to list available fields
-        verbose: Whether to enable verbose output
-    """
+    """Compare the differences between two CSV or JSON files."""
     # Set logging level based on verbosity
     if verbose:
         logger.setLevel(logging.DEBUG)

From c30ccc81c706092ffeca304adc3d5568112e9467 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 11:57:13 -0500
Subject: [PATCH 107/129] formatting fix

---
 db_diff/cli.py | 38 +++++++++++++++-----------------------
 1 file changed, 15 insertions(+), 23 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index a434031..5e59aad 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -163,29 +163,21 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         "max_content_width": TERMINAL_WIDTH
     },
     help="Compare two CSV or JSON files and identify differences.",
-    epilog="""
-Examples:
-  Basic comparison:
-    db-diff old.csv new.csv
-    
-  With specific key column:
-    db-diff old.csv new.csv --key=Id
-    
-  Output to JSON file:
-    db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json
-    
-  Compare only specific fields:
-    db-diff old.csv new.csv --fields=Id,Name,Email
-    
-  Ignore specific fields:
-    db-diff old.csv new.csv --ignorefields=LastModified,CreatedDate
-    
-  Use streaming mode for large files:
-    db-diff old.csv new.csv --streaming=normal
-    
-  List available fields:
-    db-diff old.csv new.csv --listfields
-    """,
+    epilog="\nExamples:\n"
+           "\n  Basic comparison:\n"
+           "    db-diff old.csv new.csv\n"
+           "\n  With specific key column:\n"
+           "    db-diff old.csv new.csv --key=Id\n"
+           "\n  Output to JSON file:\n"
+           "    db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json\n"
+           "\n  Compare only specific fields:\n"
+           "    db-diff old.csv new.csv --fields=Id,Name,Email\n"
+           "\n  Ignore specific fields:\n"
+           "    db-diff old.csv new.csv --ignorefields=LastModified,CreatedDate\n"
+           "\n  Use streaming mode for large files:\n"
+           "    db-diff old.csv new.csv --streaming=normal\n"
+           "\n  List available fields:\n"
+           "    db-diff old.csv new.csv --listfields",
     no_args_is_help=True,
 )
 @click.argument(

From b2247fa4591ec00b3b0cf2ced956df17432d0fcb Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 12:04:05 -0500
Subject: [PATCH 108/129] updated helptext

---
 db_diff/cli.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 5e59aad..71b3dd4 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -174,10 +174,10 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
            "    db-diff old.csv new.csv --fields=Id,Name,Email\n"
            "\n  Ignore specific fields:\n"
            "    db-diff old.csv new.csv --ignorefields=LastModified,CreatedDate\n"
-           "\n  Use streaming mode for large files:\n"
-           "    db-diff old.csv new.csv --streaming=normal\n"
            "\n  List available fields:\n"
-           "    db-diff old.csv new.csv --listfields",
+           "    db-diff old.csv new.csv --listfields"
+           "\n  Use streaming mode for large files:\n"
+           "    off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (select best mode).\n",
     no_args_is_help=True,
 )
 @click.argument(
@@ -199,7 +199,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     default="readable",
     show_default=True,
     type=click.Choice(["readable", "json", "jsonfile"], case_sensitive=False),
-    help="Output format (readable|json|jsonfile).",
+    help="Output format.",
 )
 @click.option(
     "--outfilename",
@@ -219,13 +219,13 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     "--fields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to compare (all others ignored). Cannot use with --ignorefields. Example: --fields=Id,Name,Email",
+    help="Comma-separated list of fields to compare. Cannot use with --ignorefields.",
 )
 @click.option(
     "--ignorefields",
     type=str,
     default=None,
-    help="Comma-separated list of fields to ignore. Cannot use with --fields. Example: --ignorefields=LastModifiedDate,SystemModstamp",
+    help="Comma-separated list of fields to ignore. Cannot use with --fields.",
 )
 @click.option(
     "--showunchanged",
@@ -238,7 +238,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     "input_format",
     default=None,
     type=click.Choice(list(SUPPORTED_FORMATS), case_sensitive=False),
-    help="Input format (csv|tsv|json). [default: auto-detect from file extension]",
+    help="Input format. [default: auto-detect from file extension]",
 )
 @click.option(
     "--encoding",
@@ -251,7 +251,8 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     "--streaming",
     type=click.Choice(["off", "normal", "sortedkey", "auto"], case_sensitive=False),
     default="auto",
-    help="Processing mode for large CSV/TSV files: off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (select best mode).",
+    show_default=True,
+    help="Processing mode for large CSV files.",
 )
 @click.option(
     "--listfields",

From 6224a340ca2d3083d4e9b9eda0b55096a2c334a7 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 12:04:34 -0500
Subject: [PATCH 109/129] helptext

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 71b3dd4..f24b0fb 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -175,7 +175,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
            "\n  Ignore specific fields:\n"
            "    db-diff old.csv new.csv --ignorefields=LastModified,CreatedDate\n"
            "\n  List available fields:\n"
-           "    db-diff old.csv new.csv --listfields"
+           "    db-diff old.csv new.csv --listfields\n"
            "\n  Use streaming mode for large files:\n"
            "    off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (select best mode).\n",
     no_args_is_help=True,

From ffeef236115dd55b61dbc9954675bbd6f4d3980c Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 12:05:42 -0500
Subject: [PATCH 110/129] helptext

---
 db_diff/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index f24b0fb..5ac8d97 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -164,10 +164,10 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     },
     help="Compare two CSV or JSON files and identify differences.",
     epilog="\nExamples:\n"
-           "\n  Basic comparison:\n"
+           "\n  Basic comparison:\n\n"
            "    db-diff old.csv new.csv\n"
            "\n  With specific key column:\n"
-           "    db-diff old.csv new.csv --key=Id\n"
+           "\n    db-diff old.csv new.csv --key=Id\n"
            "\n  Output to JSON file:\n"
            "    db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json\n"
            "\n  Compare only specific fields:\n"

From 2c06330b273f485fb8335de56490b77a3e1b5820 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 12:06:27 -0500
Subject: [PATCH 111/129] test

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 5ac8d97..cdfb941 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -166,7 +166,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     epilog="\nExamples:\n"
            "\n  Basic comparison:\n\n"
            "    db-diff old.csv new.csv\n"
-           "\n  With specific key column:\n"
+           "\n  With specific key column:"
            "\n    db-diff old.csv new.csv --key=Id\n"
            "\n  Output to JSON file:\n"
            "    db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json\n"

From 7e8abef3578a345d9680c6e9a5ee22c3c17694cd Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 12:07:57 -0500
Subject: [PATCH 112/129] helptext

---
 db_diff/cli.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index cdfb941..262e6c9 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -164,20 +164,20 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     },
     help="Compare two CSV or JSON files and identify differences.",
     epilog="\nExamples:\n"
-           "\n  Basic comparison:\n\n"
-           "    db-diff old.csv new.csv\n"
-           "\n  With specific key column:"
-           "\n    db-diff old.csv new.csv --key=Id\n"
-           "\n  Output to JSON file:\n"
-           "    db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json\n"
-           "\n  Compare only specific fields:\n"
-           "    db-diff old.csv new.csv --fields=Id,Name,Email\n"
-           "\n  Ignore specific fields:\n"
-           "    db-diff old.csv new.csv --ignorefields=LastModified,CreatedDate\n"
-           "\n  List available fields:\n"
-           "    db-diff old.csv new.csv --listfields\n"
-           "\n  Use streaming mode for large files:\n"
-           "    off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (select best mode).\n",
+           "\n Basic comparison:\n"
+           "\n     db-diff old.csv new.csv\n"
+           "\n With specific key column:\n"
+           "\n     db-diff old.csv new.csv --key=Id\n"
+           "\n Output to JSON file:\n"
+           "\n     db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json\n"
+           "\n Compare only specific fields:\n"
+           "\n     db-diff old.csv new.csv --fields=Id,Name,Email\n"
+           "\n Ignore specific fields:\n"
+           "\n     db-diff old.csv new.csv --ignorefields=LastModified,CreatedDate\n"
+           "\n List available fields:\n"
+           "\n     db-diff old.csv new.csv --listfields\n"
+           "\n Use streaming mode for large files:\n"
+           "\n     off (disable streaming), normal (memory efficient), sortedkey (fast with sorted keys), auto (select best mode).\n",
     no_args_is_help=True,
 )
 @click.argument(

From 76e00e72ca9e46b089f681bed1cfe60393eab572 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 13:13:26 -0500
Subject: [PATCH 113/129] added --timestamp option

---
 README.md      |  8 ++++++++
 db_diff/cli.py | 37 +++++++++++++++++++++++++++----------
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index f234899..da65f91 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,7 @@
 - Compare CSV, TSV, and JSON files for differences
 - Human-readable and machine-readable (JSON) output
 - JSON output to file with filename and path support
+- Optional timestamp in output filenames
 - Detects added, removed, and changed rows and columns
 - Supports custom key columns for row identity
 - Field inclusion/exclusion for focused diffs
@@ -279,6 +280,7 @@ A summary of key options:
 | `--output TEXT`       | Output format: `readable`, `json`, or `jsonfile` (default: readable)                         |
 | `--outfilename FILE`  | File to write JSON output to (used with `--output=jsonfile`)                                 |
 | `--outfilepath DIR`   | Directory to save the output file (used with `--output=jsonfile`)                            |
+| `--timestamp TEXT`    | Include timestamp in output filename: `true` or `false` (default: true)                      |
 | `--fields TEXT`       | Comma-separated list of fields to compare (all others ignored)                               |
 | `--ignorefields TEXT` | Comma-separated list of fields to ignore during comparison                                   |
 | `--showunchanged`     | Show all fields for changed records, not just changed fields                                 |
@@ -312,6 +314,12 @@ db-diff one.csv two.csv --key=Id --output=json
 db-diff one.csv two.csv --key=Id --output=jsonfile --outfilename=diffs.json
 ```
 
+**Save JSON output without timestamp in filename:**
+
+```sh
+db-diff one.csv two.csv --key=Id --output=jsonfile --timestamp=false
+```
+
 **Compare only specific fields:**
 
 ```sh
diff --git a/db_diff/cli.py b/db_diff/cli.py
index 262e6c9..9c39370 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -115,7 +115,8 @@ def load(filename: str, key: str, input_format: str, encoding: str = "utf-8") ->
         raise click.ClickException(f"Failed to load '{filename}': {e}")
 
 def generate_output_filename(previous: str, current: str, output_file: Optional[str], 
-                         output_path: Optional[str], timestamp: Optional[str] = None) -> str:
+                         output_path: Optional[str], timestamp: Optional[str] = None,
+                         use_timestamp: bool = True) -> str:
     """
     Generate an output filename for the diff result.
     
@@ -125,6 +126,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         output_file: User-specified output filename (optional)
         output_path: User-specified output directory (optional)
         timestamp: Timestamp to use (optional)
+        use_timestamp: Whether to include timestamp in filename (default: True)
         
     Returns:
         Full path to the output file
@@ -132,8 +134,11 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     Raises:
         click.ClickException: If the output path doesn't exist
     """
-    if timestamp is None:
-        timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    timestamp_prefix = ""
+    if use_timestamp:
+        if timestamp is None:
+            timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        timestamp_prefix = f"{timestamp}_"
     
     # Extract base filenames without extensions for the comparison files
     prev_basename = os.path.splitext(os.path.basename(previous))[0]
@@ -141,11 +146,13 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     
     if not output_file:
         # Default output filename
-        output_file = f"{timestamp}_diffs_{prev_basename}__VS__{curr_basename}.json"
+        output_file = f"{timestamp_prefix}diffs_{prev_basename}__vs__{curr_basename}.json"
     else:
-        # Append the user-specified filename to the timestamp
-        base_name = os.path.splitext(output_file)[0]  # Remove extension if present
-        output_file = f"{timestamp}_diffs_{base_name}.json"
+        # Prepend the timestamp to the user-specified filename if needed
+        output_file = f"{timestamp_prefix}diffs_{output_file}"
+        # Ensure the output file has a .json extension
+        if not output_file.endswith('.json'):
+            output_file += '.json'
     
     # Handle output path
     if output_path:
@@ -170,6 +177,8 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
            "\n     db-diff old.csv new.csv --key=Id\n"
            "\n Output to JSON file:\n"
            "\n     db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json\n"
+           "\n Output without timestamp in filename:\n"
+           "\n     db-diff old.csv new.csv --output=jsonfile --timestamp=false\n"
            "\n Compare only specific fields:\n"
            "\n     db-diff old.csv new.csv --fields=Id,Name,Email\n"
            "\n Ignore specific fields:\n"
@@ -206,7 +215,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     "output_file",
     type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
     default=None,
-    help="File to write JSON output to. [default: YYYY-MM-DD_HH-MM-SS_diffs_file1name__VS__file2name.json]",
+    help="Specify custom filename. [default: diffs_<filename1__vs__filename2>.json]",
 )
 @click.option(
     "--outfilepath",
@@ -264,11 +273,18 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     is_flag=True,
     help="Enable verbose output.",
 )
+@click.option(
+    "--timestamp",
+    type=click.Choice(["true", "false"], case_sensitive=False),
+    default="false",
+    show_default=True,
+    help="Add timestamp to filename: YYYY-MM-DD_HH-MM-SS_diffs_<filename>.json",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,
     show_unchanged, encoding, output, output_file, output_path,
-    fields, ignorefields, streaming, listfields, verbose
+    fields, ignorefields, streaming, listfields, verbose, timestamp
 ):
     """Compare the differences between two CSV or JSON files."""
     # Set logging level based on verbosity
@@ -323,7 +339,8 @@ def cli(
     # Set default output filename if needed
     if output == "jsonfile":
         try:
-            output_file = generate_output_filename(previous, current, output_file, output_path)
+            use_timestamp = timestamp.lower() == "true"
+            output_file = generate_output_filename(previous, current, output_file, output_path, use_timestamp=use_timestamp)
         except Exception as e:
             raise click.ClickException(f"Failed to generate output filename: {e}")
 

From 0e050a4a561d348661e73bdae31541d62ef1d739 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 13:17:36 -0500
Subject: [PATCH 114/129] Update README.md

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index da65f91..e2fef69 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,9 @@
 # db-diff
 
-[![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)]- **Key features:**
+[![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
+
+**Key features:**
 
 - **Custom Key Column:** Use `--key` to specify which column uniquely identifies rows.
 - **Output Formats:** Choose between human-readable (`readable`), JSON (`json`), or save JSON to a file (`jsonfile`).
@@ -10,7 +13,7 @@
 - **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
 - **List Fields:** Use `--listfields` to print available columns and exit.
 - **Timing:** Use `--time` to display how long the diff operation took.ithub.com/datsom1/db-diff/releases)
-[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
+
 
 > **db-diff** is the most advanced and user-friendly Python CLI tool and library for comparing CSV, TSV, and JSON database dumps. Designed specifically for database workflows, it delivers precise, human-readable or machine-readable diffs, supports custom key columns, handles massive files efficiently with streaming mode, and offers flexible field selection.
 

From e843227657d04dc982cbbeb5e5c3da5df7ba5b13 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 13:18:28 -0500
Subject: [PATCH 115/129] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e2fef69..05a4849 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 - **Encoding:** Specify file encoding with `--encoding`.
 - **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
 - **List Fields:** Use `--listfields` to print available columns and exit.
-- **Timing:** Use `--time` to display how long the diff operation took.ithub.com/datsom1/db-diff/releases)
+- **Timing:** Use `--time` to display how long the diff operation took.
 
 
 > **db-diff** is the most advanced and user-friendly Python CLI tool and library for comparing CSV, TSV, and JSON database dumps. Designed specifically for database workflows, it delivers precise, human-readable or machine-readable diffs, supports custom key columns, handles massive files efficiently with streaming mode, and offers flexible field selection.

From 4f210b521e6375500162c90ea55f0e2738f1eb05 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 13:19:32 -0500
Subject: [PATCH 116/129] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 05a4849..f946a23 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
 - **Custom Key Column:** Use `--key` to specify which column uniquely identifies rows.
 - **Output Formats:** Choose between human-readable (`readable`), JSON (`json`), or save JSON to a file (`jsonfile`).
 - **Field Selection:** Use `--fields` to compare only specific columns, or `--ignorefields` to exclude columns.
-- **Streaming Mode Options:** For very large files, choose from multiple streaming modes (`--streaming=auto|normal|sortedkey|off`) to optimize performance.
+- **Streaming Mode:** For very large files, choose from multiple streaming modes (`--streaming=auto|normal|sortedkey|off`) to optimize performance.
 - **Encoding:** Specify file encoding with `--encoding`.
 - **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
 - **List Fields:** Use `--listfields` to print available columns and exit.

From ffefa797d488d597989a7b017b06f347bdf71072 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 13:23:25 -0500
Subject: [PATCH 117/129] updated helptext

---
 db_diff/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 9c39370..258f30c 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -126,7 +126,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         output_file: User-specified output filename (optional)
         output_path: User-specified output directory (optional)
         timestamp: Timestamp to use (optional)
-        use_timestamp: Whether to include timestamp in filename (default: True)
+        use_timestamp: Whether to include timestamp in filename (default: False)
         
     Returns:
         Full path to the output file
@@ -208,7 +208,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     default="readable",
     show_default=True,
     type=click.Choice(["readable", "json", "jsonfile"], case_sensitive=False),
-    help="Output format.",
+    help="Output format, 'jsonfile' will save output to a file.",
 )
 @click.option(
     "--outfilename",

From 97e05284c050d48ef876a0b193c5a61bea4d7611 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 13:25:11 -0500
Subject: [PATCH 118/129] minor

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 258f30c..aca40b0 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -269,7 +269,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     help="List available fields in input files and exit.",
 )
 @click.option(
-    "--verbose", "-v",
+    "--verbose",
     is_flag=True,
     help="Enable verbose output.",
 )

From 515d3eda2f66aeeef934181d9a8ed2d860770892 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 14:23:59 -0500
Subject: [PATCH 119/129] fixed stupid code

---
 db_diff/cli.py | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index aca40b0..82566a7 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -115,8 +115,7 @@ def load(filename: str, key: str, input_format: str, encoding: str = "utf-8") ->
         raise click.ClickException(f"Failed to load '{filename}': {e}")
 
 def generate_output_filename(previous: str, current: str, output_file: Optional[str], 
-                         output_path: Optional[str], timestamp: Optional[str] = None,
-                         use_timestamp: bool = True) -> str:
+                         output_path: Optional[str], timestamp: bool = False) -> str:
     """
     Generate an output filename for the diff result.
     
@@ -125,8 +124,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         current: Path to the current file
         output_file: User-specified output filename (optional)
         output_path: User-specified output directory (optional)
-        timestamp: Timestamp to use (optional)
-        use_timestamp: Whether to include timestamp in filename (default: False)
+        timestamp: Whether to include timestamp in filename (default: False)
         
     Returns:
         Full path to the output file
@@ -135,10 +133,9 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         click.ClickException: If the output path doesn't exist
     """
     timestamp_prefix = ""
-    if use_timestamp:
-        if timestamp is None:
-            timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-        timestamp_prefix = f"{timestamp}_"
+    if timestamp:
+        timestamp_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        timestamp_prefix = f"{timestamp_str}_"
     
     # Extract base filenames without extensions for the comparison files
     prev_basename = os.path.splitext(os.path.basename(previous))[0]
@@ -148,11 +145,10 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         # Default output filename
         output_file = f"{timestamp_prefix}diffs_{prev_basename}__vs__{curr_basename}.json"
     else:
-        # Prepend the timestamp to the user-specified filename if needed
-        output_file = f"{timestamp_prefix}diffs_{output_file}"
-        # Ensure the output file has a .json extension
-        if not output_file.endswith('.json'):
-            output_file += '.json'
+        # Remove .json if present to avoid double extension
+        if output_file.lower().endswith('.json'):
+            output_file = output_file[:-5]
+        output_file = f"{timestamp_prefix}diffs_{output_file}.json"
     
     # Handle output path
     if output_path:
@@ -177,8 +173,8 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
            "\n     db-diff old.csv new.csv --key=Id\n"
            "\n Output to JSON file:\n"
            "\n     db-diff old.csv new.csv --output=jsonfile --outfilename=diff.json\n"
-           "\n Output without timestamp in filename:\n"
-           "\n     db-diff old.csv new.csv --output=jsonfile --timestamp=false\n"
+           "\n Output with timestamp in filename:\n"
+           "\n     db-diff old.csv new.csv --output=jsonfile --timestamp\n"
            "\n Compare only specific fields:\n"
            "\n     db-diff old.csv new.csv --fields=Id,Name,Email\n"
            "\n Ignore specific fields:\n"
@@ -275,9 +271,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
 )
 @click.option(
     "--timestamp",
-    type=click.Choice(["true", "false"], case_sensitive=False),
-    default="false",
-    show_default=True,
+    is_flag=True,
     help="Add timestamp to filename: YYYY-MM-DD_HH-MM-SS_diffs_<filename>.json",
 )
 @click.version_option()
@@ -339,8 +333,7 @@ def cli(
     # Set default output filename if needed
     if output == "jsonfile":
         try:
-            use_timestamp = timestamp.lower() == "true"
-            output_file = generate_output_filename(previous, current, output_file, output_path, use_timestamp=use_timestamp)
+            output_file = generate_output_filename(previous, current, output_file, output_path, timestamp)
         except Exception as e:
             raise click.ClickException(f"Failed to generate output filename: {e}")
 

From bec30c89e2245141f3614116a5a107e29a406fca Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 14:53:48 -0500
Subject: [PATCH 120/129] refactoring

---
 db_diff/cli.py | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 82566a7..c9aa58f 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -114,8 +114,7 @@ def load(filename: str, key: str, input_format: str, encoding: str = "utf-8") ->
         logger.error(f"Error loading file {filename}: {str(e)}")
         raise click.ClickException(f"Failed to load '{filename}': {e}")
 
-def generate_output_filename(previous: str, current: str, output_file: Optional[str], 
-                         output_path: Optional[str], timestamp: bool = False) -> str:
+def generate_output_filename(previous: str, current: str, output_file: Optional[str], output_path: Optional[str], timestamp: bool = False) -> str:
     """
     Generate an output filename for the diff result.
     
@@ -132,33 +131,36 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     Raises:
         click.ClickException: If the output path doesn't exist
     """
-    timestamp_prefix = ""
-    if timestamp:
-        timestamp_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-        timestamp_prefix = f"{timestamp_str}_"
-    
     # Extract base filenames without extensions for the comparison files
     prev_basename = os.path.splitext(os.path.basename(previous))[0]
     curr_basename = os.path.splitext(os.path.basename(current))[0]
     
-    if not output_file:
-        # Default output filename
-        output_file = f"{timestamp_prefix}diffs_{prev_basename}__vs__{curr_basename}.json"
-    else:
-        # Remove .json if present to avoid double extension
+    # Build the base filename
+    if output_file:
+        # Remove .json extension if present to avoid double extension
         if output_file.lower().endswith('.json'):
             output_file = output_file[:-5]
-        output_file = f"{timestamp_prefix}diffs_{output_file}.json"
+        base_filename = f"diffs_{output_file}.json"
+    else:
+        # Default output filename
+        base_filename = f"diffs_{prev_basename}__vs__{curr_basename}.json"
     
-    # Handle output path
+    # Add timestamp prefix if enabled
+    if timestamp:
+        timestamp_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        base_filename = f"{timestamp_str}_{base_filename}"
+    
+    # Resolve the final path
     if output_path:
         if not os.path.isdir(output_path):
             logger.error(f"Output path '{output_path}' does not exist")
             raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
-        output_file = os.path.join(output_path, os.path.basename(output_file))
+        full_path = os.path.join(output_path, base_filename)
+    else:
+        full_path = base_filename
     
-    logger.debug(f"Generated output filename: {output_file}")
-    return output_file
+    logger.debug(f"Generated output filename: {full_path}")
+    return full_path
 
 @click.command(
     context_settings={

From ac712c894bb98962684540148671024f58ab18d3 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 14:55:55 -0500
Subject: [PATCH 121/129] minor

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index c9aa58f..2873845 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -155,7 +155,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         if not os.path.isdir(output_path):
             logger.error(f"Output path '{output_path}' does not exist")
             raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
-        full_path = os.path.join(output_path, base_filename)
+        full_path = os.path.join(output_path, os.path.basename(base_filename))
     else:
         full_path = base_filename
     

From bf2c791367d6d4e777c46843998f643a1a393d50 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 15:04:59 -0500
Subject: [PATCH 122/129] tesr

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 2873845..c9aa58f 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -155,7 +155,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         if not os.path.isdir(output_path):
             logger.error(f"Output path '{output_path}' does not exist")
             raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
-        full_path = os.path.join(output_path, os.path.basename(base_filename))
+        full_path = os.path.join(output_path, base_filename)
     else:
         full_path = base_filename
     

From d1c34e22b22a8e5b0b2b24ecd117a7a4e2b19d80 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 15:07:19 -0500
Subject: [PATCH 123/129] test

---
 db_diff/cli.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index c9aa58f..18e1a21 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -155,6 +155,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         if not os.path.isdir(output_path):
             logger.error(f"Output path '{output_path}' does not exist")
             raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
+        print(f"base_filename: {base_filename}")
         full_path = os.path.join(output_path, base_filename)
     else:
         full_path = base_filename

From b13596182aa3056c1147e982586b3862e918afc6 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 15:10:27 -0500
Subject: [PATCH 124/129] test

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 18e1a21..0d3255d 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -134,7 +134,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     # Extract base filenames without extensions for the comparison files
     prev_basename = os.path.splitext(os.path.basename(previous))[0]
     curr_basename = os.path.splitext(os.path.basename(current))[0]
-    
+    print(f"output_file: {output_file}")
     # Build the base filename
     if output_file:
         # Remove .json extension if present to avoid double extension

From d72034ccef7a16ab8fb1ed25e306e0b9a90a1b42 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 15:16:04 -0500
Subject: [PATCH 125/129] bug fix

---
 db_diff/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 0d3255d..e2101b2 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -212,7 +212,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
 @click.option(
     "--outfilename",
     "output_file",
-    type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
+    type=str,
     default=None,
     help="Specify custom filename. [default: diffs_<filename1__vs__filename2>.json]",
 )

From dc40e2c4640edb7b6edcd6c1ea55dd5e72973e16 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 15:16:55 -0500
Subject: [PATCH 126/129] removed lines for testing

---
 db_diff/cli.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index e2101b2..ce4e8af 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -134,7 +134,7 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     # Extract base filenames without extensions for the comparison files
     prev_basename = os.path.splitext(os.path.basename(previous))[0]
     curr_basename = os.path.splitext(os.path.basename(current))[0]
-    print(f"output_file: {output_file}")
+    
     # Build the base filename
     if output_file:
         # Remove .json extension if present to avoid double extension
@@ -155,7 +155,6 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
         if not os.path.isdir(output_path):
             logger.error(f"Output path '{output_path}' does not exist")
             raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
-        print(f"base_filename: {base_filename}")
         full_path = os.path.join(output_path, base_filename)
     else:
         full_path = base_filename

From ca510a99c065e9fcbe6f977087d863abc9ba7e38 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 16:26:49 -0500
Subject: [PATCH 127/129] Update setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c822752..b415345 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import io
 import os
 
-VERSION = "4.1"
+VERSION = "4.2"
 
 def get_long_description():
     with io.open(

From 786fcdc2cba4a7aba7a79f62b207ce9e29a10bfe Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 16:28:19 -0500
Subject: [PATCH 128/129] lots of changes and improvements I believe everything
 works

---
 README.md      |  20 ++++--
 db_diff/cli.py | 181 ++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 170 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index f946a23..5dcd889 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
 
 [![Changelog](https://img.shields.io/github/v/release/datsom1/db-diff?include_prereleases&label=changelog)](https://github.com/datsom1/db-diff/releases)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/datsom1/db-diff/blob/main/LICENSE)
+[![Python Versions](https://img.shields.io/badge/python-3.6%2B-blue)](https://github.com/datsom1/db-diff)
 
 **Key features:**
 
@@ -12,7 +13,7 @@
 - **Encoding:** Specify file encoding with `--encoding`.
 - **Show Unchanged:** Use `--showunchanged` to display unchanged fields for changed rows.
 - **List Fields:** Use `--listfields` to print available columns and exit.
-- **Timing:** Use `--time` to display how long the diff operation took.
+- **Performance Metrics:** Automatically displays elapsed time for diff operations.
 
 
 > **db-diff** is the most advanced and user-friendly Python CLI tool and library for comparing CSV, TSV, and JSON database dumps. Designed specifically for database workflows, it delivers precise, human-readable or machine-readable diffs, supports custom key columns, handles massive files efficiently with streaming mode, and offers flexible field selection.
@@ -63,6 +64,11 @@ To upgrade to the latest version:
 pip install --upgrade --force-reinstall git+https://github.com/datsom1/db-diff.git
 ```
 
+### Requirements
+
+- Python 3.6 or higher
+- Dependencies are automatically installed with pip
+
 ---
 
 ## Quick Start
@@ -283,7 +289,7 @@ A summary of key options:
 | `--output TEXT`       | Output format: `readable`, `json`, or `jsonfile` (default: readable)                         |
 | `--outfilename FILE`  | File to write JSON output to (used with `--output=jsonfile`)                                 |
 | `--outfilepath DIR`   | Directory to save the output file (used with `--output=jsonfile`)                            |
-| `--timestamp TEXT`    | Include timestamp in output filename: `true` or `false` (default: true)                      |
+| `--timestamp`         | Add timestamp to filename (YYYY-MM-DD_HH-MM-SS_diffs_<filename>.json)                       |
 | `--fields TEXT`       | Comma-separated list of fields to compare (all others ignored)                               |
 | `--ignorefields TEXT` | Comma-separated list of fields to ignore during comparison                                   |
 | `--showunchanged`     | Show all fields for changed records, not just changed fields                                 |
@@ -291,7 +297,8 @@ A summary of key options:
 | `--encoding TEXT`     | Input file encoding (default: utf-8)                                                         |
 | `--streaming=MODE`    | Streaming mode for CSV/TSV files. Options: `auto` (default), `normal`, `sortedkey`, `off`    |
 | `--listfields`        | List available fields/columns in the input files and exit                                    |
-| `--verbose`, `-v`     | Enable verbose output with detailed logging                                                  |
+| `--verbose`           | Enable verbose output with detailed logging                                                  |
+| `--samplesize NUM`    | Number of rows to sample when checking if key columns are sorted (default: 1000)             |
 | `--version`           | Show the version and exit                                                                    |
 | `-h, --help`          | Show help message and exit                                                                   |
 
@@ -317,10 +324,10 @@ db-diff one.csv two.csv --key=Id --output=json
 db-diff one.csv two.csv --key=Id --output=jsonfile --outfilename=diffs.json
 ```
 
-**Save JSON output without timestamp in filename:**
+**Save JSON output with timestamp in filename:**
 
 ```sh
-db-diff one.csv two.csv --key=Id --output=jsonfile --timestamp=false
+db-diff one.csv two.csv --key=Id --output=jsonfile --timestamp
 ```
 
 **Compare only specific fields:**
@@ -349,6 +356,9 @@ db-diff large1.csv large2.csv --key=Id --streaming=sortedkey
 
 # Disable streaming regardless of file size
 db-diff large1.csv large2.csv --key=Id --streaming=off
+
+# Adjust the sample size for key column sorting detection
+db-diff large1.csv large2.csv --key=Id --samplesize=5000
 ```
 
 The streaming modes work with CSV/TSV files only. The `auto` mode will automatically detect if the key columns are sorted and select the most efficient streaming mode for your data.
diff --git a/db_diff/cli.py b/db_diff/cli.py
index ce4e8af..81ad32d 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -12,7 +12,7 @@
 import logging
 import datetime
 import traceback
-from typing import Dict, Set, List, Optional, Any
+from typing import Dict, Set, List, Optional, Any, Tuple, Callable, Union, Iterator
 
 import click
 
@@ -102,14 +102,30 @@ def load(filename: str, key: str, input_format: str, encoding: str = "utf-8") ->
     try:
         if input_format == "json":
             with open(filename, encoding=encoding) as f:
-                data = load_json(f, key=key)
-                logger.debug(f"Loaded {len(data)} rows from JSON file")
-                return data
+                try:
+                    data = load_json(f, key=key)
+                    logger.debug(f"Loaded {len(data)} rows from JSON file")
+                    return data
+                except ValueError as e:
+                    raise click.ClickException(f"Invalid JSON format in '{filename}': {e}")
+                except KeyError as e:
+                    raise click.ClickException(f"Key error in JSON file '{filename}': {e}")
         else:
             with open(filename, newline="", encoding=encoding) as f:
-                data = load_csv(f, key=key, dialect=DIALECTS.get(input_format, "excel"))
-                logger.debug(f"Loaded {len(data)} rows from CSV/TSV file")
-                return data
+                try:
+                    data = load_csv(f, key=key, dialect=DIALECTS.get(input_format, "excel"))
+                    logger.debug(f"Loaded {len(data)} rows from CSV/TSV file")
+                    return data
+                except csv.Error as e:
+                    raise click.ClickException(f"CSV parsing error in '{filename}': {e}")
+                except KeyError as e:
+                    raise click.ClickException(f"Key column '{key}' not found in all rows in '{filename}'")
+    except FileNotFoundError:
+        raise click.ClickException(f"File not found: '{filename}'")
+    except PermissionError:
+        raise click.ClickException(f"Permission denied when accessing '{filename}'")
+    except UnicodeDecodeError:
+        raise click.ClickException(f"Failed to decode '{filename}' with encoding '{encoding}'. Try a different encoding.")
     except Exception as e:
         logger.error(f"Error loading file {filename}: {str(e)}")
         raise click.ClickException(f"Failed to load '{filename}': {e}")
@@ -162,6 +178,59 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     logger.debug(f"Generated output filename: {full_path}")
     return full_path
 
+def validate_options(fields, ignorefields, input_format, encoding, output, output_file, output_path) -> None:
+    """
+    Validate command-line options for compatibility and correctness.
+    
+    Args:
+        fields: Comma-separated list of fields to compare
+        ignorefields: Comma-separated list of fields to ignore
+        input_format: Input file format
+        encoding: File encoding
+        output: Output format
+        output_file: Output filename
+        output_path: Output directory path
+        
+    Raises:
+        click.ClickException: If options are invalid or incompatible
+    """
+    # Validate mutually exclusive options
+    if fields and ignorefields:
+        raise click.ClickException("You cannot use --fields and --ignorefields at the same time.")
+    
+    # Validate input format
+    if input_format and input_format not in SUPPORTED_FORMATS:
+        raise click.ClickException(
+            f"Unsupported format: {input_format}. "
+            f"Supported formats: {', '.join(SUPPORTED_FORMATS)}"
+        )
+    
+    # Validate encoding (try to decode a simple string to verify)
+    if encoding:
+        try:
+            "test".encode().decode(encoding)
+        except LookupError:
+            raise click.ClickException(f"Unknown encoding: {encoding}")
+    
+    # Check output path exists if specified
+    if output_path and not os.path.isdir(output_path):
+        raise click.ClickException(f"Output path '{output_path}' does not exist or is not a directory.")
+    
+    # Check if output file is specified when using jsonfile output
+    if output == "jsonfile" and not output_file and not output_path:
+        logger.warning("No output filename or path specified for jsonfile output, using default.")
+        
+    # For fields and ignorefields, check for empty fields
+    if fields:
+        field_list = [f.strip() for f in fields.split(",")]
+        if "" in field_list:
+            raise click.ClickException("Empty field name in --fields list")
+    
+    if ignorefields:
+        ignored_fields = [f.strip() for f in ignorefields.split(",")]
+        if "" in ignored_fields:
+            raise click.ClickException("Empty field name in --ignorefields list")
+
 @click.command(
     context_settings={
         "help_option_names": ["-h", "--help"],
@@ -276,12 +345,32 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     is_flag=True,
     help="Add timestamp to filename: YYYY-MM-DD_HH-MM-SS_diffs_<filename>.json",
 )
+@click.option(
+    "--samplesize",
+    type=int,
+    default=1000,
+    show_default=True,
+    help="Number of rows to sample when checking if key columns are sorted.",
+)
 @click.version_option()
 def cli(
-    previous, current, key, input_format,
-    show_unchanged, encoding, output, output_file, output_path,
-    fields, ignorefields, streaming, listfields, verbose, timestamp
-):
+    previous: str, 
+    current: str, 
+    key: Optional[str] = None, 
+    input_format: Optional[str] = None,
+    show_unchanged: bool = False, 
+    encoding: str = "utf-8", 
+    output: str = "readable", 
+    output_file: Optional[str] = None, 
+    output_path: Optional[str] = None,
+    fields: Optional[str] = None, 
+    ignorefields: Optional[str] = None, 
+    streaming: str = "auto", 
+    listfields: bool = False, 
+    verbose: bool = False, 
+    timestamp: bool = False,
+    samplesize: int = 1000,
+) -> None:
     """Compare the differences between two CSV or JSON files."""
     # Set logging level based on verbosity
     if verbose:
@@ -292,6 +381,9 @@ def cli(
     
     click.echo(f"Starting diff: '{previous}' vs '{current}'...")
 
+    # Validate command-line options
+    validate_options(fields, ignorefields, input_format, encoding, output, output_file, output_path)
+
     # Detect file format
     detected_format = input_format or os.path.splitext(previous)[1].lower().lstrip('.')
     if detected_format not in SUPPORTED_FORMATS:
@@ -300,6 +392,12 @@ def cli(
             f"Supported formats are: {', '.join(SUPPORTED_FORMATS)}"
         )
 
+    # Validate command-line options
+    try:
+        validate_options(fields, ignorefields, detected_format, encoding, output, output_file, output_path)
+    except Exception as e:
+        raise click.ClickException(f"Invalid command-line options: {e}")
+
     # List fields and exit
     if listfields:
         try:
@@ -338,13 +436,16 @@ def cli(
             output_file = generate_output_filename(previous, current, output_file, output_path, timestamp)
         except Exception as e:
             raise click.ClickException(f"Failed to generate output filename: {e}")
-
-    # Detect key column before loading data
+    # Detect key column from the first row of a CSV/TSV/JSON file
     if key is None:
-        key = sniff_key_column(previous, detected_format, encoding)
-        if not key:
-            raise click.ClickException("Could not determine key column from the previous file.")
-        click.echo(f"Using '{key}' as the key column")
+        try:
+            key = sniff_key_column(previous, detected_format, encoding)
+            if not key:
+                raise click.ClickException("Could not determine key column from the previous file.")
+            click.echo(f"Using '{key}' as the key column")
+        except Exception as e:
+            logger.error(f"Error detecting key column: {str(e)}")
+            raise click.ClickException(f"Failed to detect key column: {e}. Please specify a key column with --key.")
 
     # Enforce mutual exclusivity
     if fields and ignorefields:
@@ -375,27 +476,55 @@ def cli(
                 use_streaming = "sortedkey"
         elif streaming == "auto":
             # Auto mode - determine the best streaming mode based on file properties
+            prev_size = os.path.getsize(previous)
+            curr_size = os.path.getsize(current)
+            total_size = prev_size + curr_size
+            
+            # Get available memory
+            try:
+                import psutil
+                available_memory = psutil.virtual_memory().available
+                memory_threshold = available_memory * 0.2  # Use at most 20% of available memory
+                logger.debug(f"Auto streaming mode: Available memory: {available_memory/1024/1024:.1f} MB, File sizes: {prev_size/1024/1024:.1f} MB + {curr_size/1024/1024:.1f} MB")
+            except ImportError:
+                # If psutil is not available, use a fixed threshold
+                memory_threshold = 500 * 1024 * 1024  # 500MB threshold
+                logger.debug(f"Auto streaming mode: psutil not available, using fixed threshold of {memory_threshold/1024/1024:.1f} MB")
+            
             if detected_format not in ("csv", "tsv"):
                 logger.debug("Auto streaming mode: Not CSV/TSV format, disabling streaming")
                 use_streaming = None
             elif fields:  # If specific fields are selected, we need to read everything, so streaming won't help much
                 logger.debug("Auto streaming mode: Field filtering active, disabling streaming")
                 use_streaming = None
-            elif os.path.getsize(previous) <= 100*1024*1024:  # Less than 10MB
+            elif total_size <= 10*1024*1024:  # Less than 10MB total
                 logger.debug("Auto streaming mode: Small file size, disabling streaming")
                 use_streaming = None
-            else:
+            elif total_size > memory_threshold:
+                # Files are too large for memory, use streaming regardless
+                logger.debug("Auto streaming mode: Files too large for memory, forcing streaming mode")
+                
                 # Check if key column is sorted in both files
-                logger.debug("Auto streaming mode: Checking if key columns are sorted")
-                if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel")) and
-                    is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"))):
+                if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize) and
+                    is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize)):
+                    logger.info("Auto streaming mode: Key columns appear to be sorted, using sorted key streaming mode")
+                    use_streaming = "sortedkey"
+                    click.echo("Key columns appear to be sorted, using fast streaming mode...")
+                else:
+                    logger.info(f"Auto streaming mode: Large files ({total_size/1024/1024:.1f} MB total), using normal streaming mode")
+                    use_streaming = "normal"
+                    click.echo("Files are large, automatically enabling streaming mode for better memory usage...")
+            else:
+                # Files could fit in memory, but check if sorting would be beneficial
+                if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize) and
+                    is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize)):
                     logger.info("Auto streaming mode: Key columns appear to be sorted, using sorted key streaming mode")
                     use_streaming = "sortedkey"
                     click.echo("Key columns appear to be sorted, using fast streaming mode...")
-                elif os.path.getsize(previous) > 100*1024*1024:  # >100MB
-                    logger.info(f"Auto streaming mode: Large file size ({os.path.getsize(previous)/1024/1024:.1f} MB), using normal streaming mode")
+                elif total_size > memory_threshold * 0.5:  # If using more than 50% of memory threshold
+                    logger.info(f"Auto streaming mode: Moderately large files ({total_size/1024/1024:.1f} MB), using normal streaming mode")
                     use_streaming = "normal"
-                    click.echo("File size is large, automatically enabling streaming mode for better performance and lower memory usage...")
+                    click.echo("Files are moderately large, using streaming mode for better memory efficiency...")
         
         # Execute the appropriate comparison method
         if use_streaming == "sortedkey":
@@ -487,4 +616,4 @@ def cli(
         logger.debug(f"Memory usage: {memory_mb:.2f} MB")
 
 if __name__ == "__main__":
-    cli()
+    cli()
\ No newline at end of file

From 4c9e200983cfe4e81b53759a941ea6b7b8c415b2 Mon Sep 17 00:00:00 2001
From: Thomas Coyle <37991982+datsom1@users.noreply.github.com>
Date: Tue, 1 Jul 2025 16:59:13 -0500
Subject: [PATCH 129/129] reorganized

---
 db_diff/cli.py | 532 +++++++++++++++++++++++++++++++------------------
 1 file changed, 342 insertions(+), 190 deletions(-)

diff --git a/db_diff/cli.py b/db_diff/cli.py
index 81ad32d..b84373b 100644
--- a/db_diff/cli.py
+++ b/db_diff/cli.py
@@ -4,35 +4,46 @@
 This module provides the command-line interface for comparing CSV and JSON files.
 """
 
-import os
+#------------------------------------------------------------------------------
+# IMPORTS
+#------------------------------------------------------------------------------
+# Standard library imports
 import csv
+import datetime
 import json
-import time
-import shutil
 import logging
-import datetime
+import os
+import shutil
+import time
 import traceback
-from typing import Dict, Set, List, Optional, Any, Tuple, Callable, Union, Iterator
+from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Union
 
+# Third-party imports
 import click
 
+# Local imports
 from . import (
-    load_csv, 
-    load_json, 
-    compare, 
-    human_text, 
-    streaming_compare_csv,
-    streaming_compare_csv_sorted,
+    compare,
+    human_text,
     is_key_column_sorted,
-    logger
+    load_csv,
+    load_json,
+    logger,
+    streaming_compare_csv,
+    streaming_compare_csv_sorted
 )
 
-# Configure logging with click integration
+#------------------------------------------------------------------------------
+# LOGGING SETUP
+#------------------------------------------------------------------------------
 logger = logging.getLogger("db_diff")
 
 # Get terminal width for better help formatting
 TERMINAL_WIDTH = shutil.get_terminal_size().columns
 
+#------------------------------------------------------------------------------
+# CONSTANTS
+#------------------------------------------------------------------------------
 # Known CSV dialects
 DIALECTS = {
     "csv": "excel",
@@ -42,6 +53,9 @@
 # Supported input formats
 SUPPORTED_FORMATS = ("csv", "tsv", "json")
 
+#------------------------------------------------------------------------------
+# FILE DETECTION AND LOADING FUNCTIONS
+#------------------------------------------------------------------------------
 def sniff_key_column(filename: str, input_format: str, encoding: str = "utf-8") -> Optional[str]:
     """
     Detect the key column from the first row of a CSV/TSV/JSON file.
@@ -130,6 +144,9 @@ def load(filename: str, key: str, input_format: str, encoding: str = "utf-8") ->
         logger.error(f"Error loading file {filename}: {str(e)}")
         raise click.ClickException(f"Failed to load '{filename}': {e}")
 
+#------------------------------------------------------------------------------
+# OUTPUT AND VALIDATION FUNCTIONS
+#------------------------------------------------------------------------------
 def generate_output_filename(previous: str, current: str, output_file: Optional[str], output_path: Optional[str], timestamp: bool = False) -> str:
     """
     Generate an output filename for the diff result.
@@ -178,7 +195,15 @@ def generate_output_filename(previous: str, current: str, output_file: Optional[
     logger.debug(f"Generated output filename: {full_path}")
     return full_path
 
-def validate_options(fields, ignorefields, input_format, encoding, output, output_file, output_path) -> None:
+def validate_options(
+    fields: Optional[str], 
+    ignorefields: Optional[str], 
+    input_format: Optional[str], 
+    encoding: str, 
+    output: str, 
+    output_file: Optional[str], 
+    output_path: Optional[str]
+) -> None:
     """
     Validate command-line options for compatibility and correctness.
     
@@ -231,6 +256,280 @@ def validate_options(fields, ignorefields, input_format, encoding, output, outpu
         if "" in ignored_fields:
             raise click.ClickException("Empty field name in --ignorefields list")
 
+def list_available_fields(filename: str, detected_format: str, encoding: str) -> None:
+    """
+    List available fields in the input file and exit.
+    
+    Args:
+        filename: Path to the input file
+        detected_format: Format of the input file
+        encoding: File encoding
+    
+    Raises:
+        click.ClickException: If listing fields fails
+    """
+    try:
+        if detected_format == "json":
+            with open(filename, encoding=encoding) as f:
+                data = json.load(f)
+                if isinstance(data, list) and data:
+                    all_fields = set()
+                    # Collect all unique fields across the first 10 records
+                    for i, item in enumerate(data[:10]):
+                        all_fields.update(item.keys())
+                    
+                    click.echo(f"Available fields in '{os.path.basename(filename)}' ({len(all_fields)} total):")
+                    for field in sorted(all_fields):
+                        click.echo(f"  {field}")
+                else:
+                    click.echo("No fields found: JSON file does not contain a list of objects")
+        else:
+            with open(filename, newline="", encoding=encoding) as f:
+                reader = csv.reader(f, dialect=DIALECTS.get(detected_format, "excel"))
+                headings = next(reader, None)
+                if headings:
+                    click.echo(f"Available fields in '{os.path.basename(filename)}' ({len(headings)} total):")
+                    for field in headings:
+                        click.echo(f"  {field}")
+                else:
+                    click.echo("No fields found: CSV file appears to be empty")
+    except Exception as e:
+        logger.error(f"Error listing fields: {str(e)}")
+        raise click.ClickException(f"Could not list fields: {e}")
+
+#------------------------------------------------------------------------------
+# STREAMING MODE HELPER FUNCTIONS
+#------------------------------------------------------------------------------
+def determine_streaming_mode(
+    streaming: str, 
+    detected_format: str, 
+    previous: str, 
+    current: str, 
+    key: str, 
+    fields: Optional[str],
+    encoding: str,
+    samplesize: int
+) -> Optional[str]:
+    """
+    Determine the appropriate streaming mode based on file properties and user settings.
+    
+    Args:
+        streaming: User-specified streaming mode
+        detected_format: Format of the input files
+        previous: Path to the previous file
+        current: Path to the current file
+        key: Key column name
+        fields: Comma-separated list of fields to compare
+        encoding: File encoding
+        samplesize: Number of rows to sample for sorting check
+        
+    Returns:
+        The streaming mode to use ('normal', 'sortedkey', or None)
+    """
+    if streaming == "off":
+        logger.debug("Streaming mode manually disabled")
+        return None
+    elif streaming in ("normal", "sortedkey"):
+        if detected_format not in ("csv", "tsv"):
+            logger.error("Streaming mode requested but file format is not CSV/TSV")
+            raise click.ClickException("Streaming modes only work with CSV/TSV files.")
+        if streaming == "sortedkey":
+            logger.debug("Using sorted key streaming mode")
+            return "sortedkey"
+        return "normal"
+    elif streaming == "auto":
+        # Auto mode - determine the best streaming mode based on file properties
+        prev_size = os.path.getsize(previous)
+        curr_size = os.path.getsize(current)
+        total_size = prev_size + curr_size
+        
+        # Get available memory
+        try:
+            import psutil
+            available_memory = psutil.virtual_memory().available
+            memory_threshold = available_memory * 0.2  # Use at most 20% of available memory
+            logger.debug(f"Auto streaming mode: Available memory: {available_memory/1024/1024:.1f} MB, File sizes: {prev_size/1024/1024:.1f} MB + {curr_size/1024/1024:.1f} MB")
+        except ImportError:
+            # If psutil is not available, use a fixed threshold
+            memory_threshold = 500 * 1024 * 1024  # 500MB threshold
+            logger.debug(f"Auto streaming mode: psutil not available, using fixed threshold of {memory_threshold/1024/1024:.1f} MB")
+        
+        if detected_format not in ("csv", "tsv"):
+            logger.debug("Auto streaming mode: Not CSV/TSV format, disabling streaming")
+            return None
+        elif fields:  # If specific fields are selected, we need to read everything, so streaming won't help much
+            logger.debug("Auto streaming mode: Field filtering active, disabling streaming")
+            return None
+        elif total_size <= 10*1024*1024:  # Less than 10MB total
+            logger.debug("Auto streaming mode: Small file size, disabling streaming")
+            return None
+        elif total_size > memory_threshold:
+            # Files are too large for memory, use streaming regardless
+            logger.debug("Auto streaming mode: Files too large for memory, forcing streaming mode")
+            
+            # Check if key column is sorted in both files
+            if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize) and
+                is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize)):
+                logger.info("Auto streaming mode: Key columns appear to be sorted, using sorted key streaming mode")
+                click.echo("Key columns appear to be sorted, using fast streaming mode...")
+                return "sortedkey"
+            else:
+                logger.info(f"Auto streaming mode: Large files ({total_size/1024/1024:.1f} MB total), using normal streaming mode")
+                click.echo("Files are large, automatically enabling streaming mode for better memory usage...")
+                return "normal"
+        else:
+            # Files could fit in memory, but check if sorting would be beneficial
+            if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize) and
+                is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize)):
+                logger.info("Auto streaming mode: Key columns appear to be sorted, using sorted key streaming mode")
+                click.echo("Key columns appear to be sorted, using fast streaming mode...")
+                return "sortedkey"
+            elif total_size > memory_threshold * 0.5:  # If using more than 50% of memory threshold
+                logger.info(f"Auto streaming mode: Moderately large files ({total_size/1024/1024:.1f} MB), using normal streaming mode")
+                click.echo("Files are moderately large, using streaming mode for better memory efficiency...")
+                return "normal"
+    
+    return None  # Default if no condition is met
+
+def execute_comparison(
+    comparison_mode: Optional[str],
+    previous: str,
+    current: str,
+    key: str,
+    fields_set: Optional[Set[str]],
+    ignorefields_set: Optional[Set[str]],
+    show_unchanged: bool,
+    encoding: str,
+    detected_format: str
+) -> Dict:
+    """
+    Execute the appropriate comparison method based on the selected mode.
+    
+    Args:
+        comparison_mode: Streaming mode to use ('normal', 'sortedkey', or None)
+        previous: Path to the previous file
+        current: Path to the current file
+        key: Key column name
+        fields_set: Set of fields to compare
+        ignorefields_set: Set of fields to ignore
+        show_unchanged: Whether to show unchanged fields
+        encoding: File encoding
+        detected_format: Format of the input files
+        
+    Returns:
+        Dictionary containing diff results
+    """
+    if comparison_mode == "sortedkey":
+        click.echo("Using sorted key streaming mode for comparison...")
+        return streaming_compare_csv_sorted(
+            previous,
+            current,
+            key=key,
+            compare_columns=fields_set,
+            ignorefields=ignorefields_set,
+            encoding=encoding,
+            dialect=DIALECTS.get(detected_format, "excel"),
+        )
+    elif comparison_mode == "normal":
+        click.echo("Using normal streaming mode for comparison...")
+        return streaming_compare_csv(
+            previous,
+            current,
+            key=key,
+            compare_columns=fields_set,
+            ignorefields=ignorefields_set,
+            encoding=encoding,
+            dialect=DIALECTS.get(detected_format, "excel"),
+        )
+    else:
+        click.echo("Loading previous data...")
+        previous_data = load(previous, key, detected_format, encoding)
+        click.echo(f"Loaded {len(previous_data)} rows from previous file")
+        
+        click.echo("Loading current data...")
+        current_data = load(current, key, detected_format, encoding)
+        click.echo(f"Loaded {len(current_data)} rows from current file")
+        
+        click.echo("Comparing data...")
+        return compare(
+            previous_data,
+            current_data,
+            show_unchanged,
+            fields=fields_set,
+            ignorefields=ignorefields_set,
+        )
+
+def process_diff_output(
+    diff: Dict, 
+    output: str, 
+    output_file: Optional[str], 
+    encoding: str,
+    key: str
+) -> None:
+    """
+    Process and output the diff results in the requested format.
+    
+    Args:
+        diff: Dictionary containing diff results
+        output: Output format ('readable', 'json', or 'jsonfile')
+        output_file: Path to output file (for 'jsonfile' output)
+        encoding: File encoding
+        key: Key column name
+    """
+    if output == "json":
+        click.echo("Generating JSON output...")
+        json_output = json.dumps(diff, indent=2)
+        print(json_output)
+        click.echo("Finished. Output printed as JSON to terminal.")
+    elif output == "jsonfile":
+        click.echo(f"Saving JSON output to {output_file}...")
+        with open(output_file, "w", encoding=encoding) as f:
+            json.dump(diff, f, indent=2)
+        click.echo(f"Finished. Output saved to file: {output_file}")
+    else:
+        click.echo("Generating human-readable output...")
+        human_output = human_text(diff, key)
+        print(human_output)
+        click.echo("Finished. Output printed in readable format to terminal.")
+
+def print_summary(diff: Dict, elapsed_time: float, verbose: bool) -> None:
+    """
+    Print summary information about the comparison results.
+    
+    Args:
+        diff: Dictionary containing diff results
+        elapsed_time: Time taken for the comparison (seconds)
+        verbose: Whether to include verbose output
+    """
+    # Display the elapsed time
+    click.echo(f"\nElapsed time: {elapsed_time:.3f} seconds")
+    
+    # Display summary statistics
+    summary = []
+    for stat_key in ["columns_added", "columns_removed", "added", "removed", "changed"]:
+        if diff[stat_key]:
+            count = len(diff[stat_key])
+            label = stat_key.replace('_', ' ')
+            summary.append(f"{count} {label}")
+    
+    if summary:
+        click.echo(f"Summary: {', '.join(summary)}")
+    else:
+        click.echo("No differences found")
+        
+    # Display memory usage if verbose
+    if verbose:
+        try:
+            import psutil
+            process = psutil.Process(os.getpid())
+            memory_mb = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"Memory usage: {memory_mb:.2f} MB")
+        except ImportError:
+            logger.debug("psutil not available, memory usage stats skipped")
+
+#------------------------------------------------------------------------------
+# COMMAND LINE INTERFACE
+#------------------------------------------------------------------------------
 @click.command(
     context_settings={
         "help_option_names": ["-h", "--help"],
@@ -381,9 +680,6 @@ def cli(
     
     click.echo(f"Starting diff: '{previous}' vs '{current}'...")
 
-    # Validate command-line options
-    validate_options(fields, ignorefields, input_format, encoding, output, output_file, output_path)
-
     # Detect file format
     detected_format = input_format or os.path.splitext(previous)[1].lower().lstrip('.')
     if detected_format not in SUPPORTED_FORMATS:
@@ -400,34 +696,7 @@ def cli(
 
     # List fields and exit
     if listfields:
-        try:
-            if detected_format == "json":
-                with open(previous, encoding=encoding) as f:
-                    data = json.load(f)
-                    if isinstance(data, list) and data:
-                        all_fields = set()
-                        # Collect all unique fields across the first 10 records
-                        for i, item in enumerate(data[:10]):
-                            all_fields.update(item.keys())
-                        
-                        click.echo(f"Available fields in '{os.path.basename(previous)}' ({len(all_fields)} total):")
-                        for field in sorted(all_fields):
-                            click.echo(f"  {field}")
-                    else:
-                        click.echo("No fields found: JSON file does not contain a list of objects")
-            else:
-                with open(previous, newline="", encoding=encoding) as f:
-                    reader = csv.reader(f, dialect=DIALECTS.get(detected_format, "excel"))
-                    headings = next(reader, None)
-                    if headings:
-                        click.echo(f"Available fields in '{os.path.basename(previous)}' ({len(headings)} total):")
-                        for field in headings:
-                            click.echo(f"  {field}")
-                    else:
-                        click.echo("No fields found: CSV file appears to be empty")
-        except Exception as e:
-            logger.error(f"Error listing fields: {str(e)}")
-            raise click.ClickException(f"Could not list fields: {e}")
+        list_available_fields(previous, detected_format, encoding)
         return
 
     # Set default output filename if needed
@@ -436,6 +705,7 @@ def cli(
             output_file = generate_output_filename(previous, current, output_file, output_path, timestamp)
         except Exception as e:
             raise click.ClickException(f"Failed to generate output filename: {e}")
+    
     # Detect key column from the first row of a CSV/TSV/JSON file
     if key is None:
         try:
@@ -447,10 +717,6 @@ def cli(
             logger.error(f"Error detecting key column: {str(e)}")
             raise click.ClickException(f"Failed to detect key column: {e}. Please specify a key column with --key.")
 
-    # Enforce mutual exclusivity
-    if fields and ignorefields:
-        raise click.ClickException("You cannot use --fields and --ignorefields at the same time.")
-
     # Parse fields/ignorefields into sets
     fields_set = set(f.strip() for f in fields.split(",")) if fields else None
     ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
@@ -459,130 +725,33 @@ def cli(
     start = time.time()
 
     try:
-        # --- Determine comparison method ---
-        use_streaming = "normal"  # Default streaming mode if enabled
-        use_streaming_sorted = False  # Whether to use the sorted key streaming mode
-        
-        # Check if streaming mode is appropriate
-        if streaming == "off":
-            logger.debug("Streaming mode manually disabled")
-            use_streaming = None
-        elif streaming in ("normal", "sortedkey"):
-            if detected_format not in ("csv", "tsv"):
-                logger.error("Streaming mode requested but file format is not CSV/TSV")
-                raise click.ClickException("Streaming modes only work with CSV/TSV files.")
-            if streaming == "sortedkey":
-                logger.debug("Using sorted key streaming mode")
-                use_streaming = "sortedkey"
-        elif streaming == "auto":
-            # Auto mode - determine the best streaming mode based on file properties
-            prev_size = os.path.getsize(previous)
-            curr_size = os.path.getsize(current)
-            total_size = prev_size + curr_size
-            
-            # Get available memory
-            try:
-                import psutil
-                available_memory = psutil.virtual_memory().available
-                memory_threshold = available_memory * 0.2  # Use at most 20% of available memory
-                logger.debug(f"Auto streaming mode: Available memory: {available_memory/1024/1024:.1f} MB, File sizes: {prev_size/1024/1024:.1f} MB + {curr_size/1024/1024:.1f} MB")
-            except ImportError:
-                # If psutil is not available, use a fixed threshold
-                memory_threshold = 500 * 1024 * 1024  # 500MB threshold
-                logger.debug(f"Auto streaming mode: psutil not available, using fixed threshold of {memory_threshold/1024/1024:.1f} MB")
-            
-            if detected_format not in ("csv", "tsv"):
-                logger.debug("Auto streaming mode: Not CSV/TSV format, disabling streaming")
-                use_streaming = None
-            elif fields:  # If specific fields are selected, we need to read everything, so streaming won't help much
-                logger.debug("Auto streaming mode: Field filtering active, disabling streaming")
-                use_streaming = None
-            elif total_size <= 10*1024*1024:  # Less than 10MB total
-                logger.debug("Auto streaming mode: Small file size, disabling streaming")
-                use_streaming = None
-            elif total_size > memory_threshold:
-                # Files are too large for memory, use streaming regardless
-                logger.debug("Auto streaming mode: Files too large for memory, forcing streaming mode")
-                
-                # Check if key column is sorted in both files
-                if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize) and
-                    is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize)):
-                    logger.info("Auto streaming mode: Key columns appear to be sorted, using sorted key streaming mode")
-                    use_streaming = "sortedkey"
-                    click.echo("Key columns appear to be sorted, using fast streaming mode...")
-                else:
-                    logger.info(f"Auto streaming mode: Large files ({total_size/1024/1024:.1f} MB total), using normal streaming mode")
-                    use_streaming = "normal"
-                    click.echo("Files are large, automatically enabling streaming mode for better memory usage...")
-            else:
-                # Files could fit in memory, but check if sorting would be beneficial
-                if (is_key_column_sorted(previous, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize) and
-                    is_key_column_sorted(current, key, encoding=encoding, dialect=DIALECTS.get(detected_format, "excel"), sample_size=samplesize)):
-                    logger.info("Auto streaming mode: Key columns appear to be sorted, using sorted key streaming mode")
-                    use_streaming = "sortedkey"
-                    click.echo("Key columns appear to be sorted, using fast streaming mode...")
-                elif total_size > memory_threshold * 0.5:  # If using more than 50% of memory threshold
-                    logger.info(f"Auto streaming mode: Moderately large files ({total_size/1024/1024:.1f} MB), using normal streaming mode")
-                    use_streaming = "normal"
-                    click.echo("Files are moderately large, using streaming mode for better memory efficiency...")
+        # Determine comparison method
+        streaming_mode = determine_streaming_mode(
+            streaming, 
+            detected_format, 
+            previous, 
+            current, 
+            key, 
+            fields,
+            encoding,
+            samplesize
+        )
         
         # Execute the appropriate comparison method
-        if use_streaming == "sortedkey":
-            click.echo("Using sorted key streaming mode for comparison...")
-            diff = streaming_compare_csv_sorted(
-                previous,
-                current,
-                key=key,
-                compare_columns=fields_set,
-                ignorefields=ignorefields_set,
-                encoding=encoding,
-                dialect=DIALECTS.get(detected_format, "excel"),
-            )
-        elif use_streaming == "normal":
-            click.echo("Using normal streaming mode for comparison...")
-            diff = streaming_compare_csv(
-                previous,
-                current,
-                key=key,
-                compare_columns=fields_set,
-                ignorefields=ignorefields_set,
-                encoding=encoding,
-                dialect=DIALECTS.get(detected_format, "excel"),
-            )
-        else:
-            click.echo("Loading previous data...")
-            previous_data = load(previous, key, detected_format, encoding)
-            click.echo(f"Loaded {len(previous_data)} rows from previous file")
-            
-            click.echo("Loading current data...")
-            current_data = load(current, key, detected_format, encoding)
-            click.echo(f"Loaded {len(current_data)} rows from current file")
-            
-            click.echo("Comparing data...")
-            diff = compare(
-                previous_data,
-                current_data,
-                show_unchanged,
-                fields=fields_set,
-                ignorefields=ignorefields_set,
-            )
+        diff = execute_comparison(
+            streaming_mode,
+            previous,
+            current,
+            key,
+            fields_set,
+            ignorefields_set,
+            show_unchanged,
+            encoding,
+            detected_format
+        )
 
         # Generate output
-        if output == "json":
-            click.echo("Generating JSON output...")
-            json_output = json.dumps(diff, indent=2)
-            print(json_output)
-            click.echo("Finished. Output printed as JSON to terminal.")
-        elif output == "jsonfile":
-            click.echo(f"Saving JSON output to {output_file}...")
-            with open(output_file, "w", encoding=encoding) as f:
-                json.dump(diff, f, indent=2)
-            click.echo(f"Finished. Output saved to file: {output_file}")
-        else:
-            click.echo("Generating human-readable output...")
-            human_output = human_text(diff, key)
-            print(human_output)
-            click.echo("Finished. Output printed in readable format to terminal.")
+        process_diff_output(diff, output, output_file, encoding, key)
     
     except Exception as e:
         logger.error(f"Error during comparison: {str(e)}")
@@ -591,29 +760,12 @@ def cli(
             logger.debug(f"Full traceback:\n{error_traceback}")
         raise click.ClickException(str(e))
 
-    # Always display the elapsed time
+    # Calculate elapsed time
     end = time.time()
-    click.echo(f"\nElapsed time: {end - start:.3f} seconds")
-    
-    # Display summary statistics
-    summary = []
-    for stat_key in ["columns_added", "columns_removed", "added", "removed", "changed"]:
-        if diff[stat_key]:
-            count = len(diff[stat_key])
-            label = stat_key.replace('_', ' ')
-            summary.append(f"{count} {label}")
+    elapsed_time = end - start
     
-    if summary:
-        click.echo(f"Summary: {', '.join(summary)}")
-    else:
-        click.echo("No differences found")
-        
-    # Display memory usage if verbose
-    if verbose:
-        import psutil
-        process = psutil.Process(os.getpid())
-        memory_mb = process.memory_info().rss / 1024 / 1024
-        logger.debug(f"Memory usage: {memory_mb:.2f} MB")
+    # Print summary
+    print_summary(diff, elapsed_time, verbose)
 
 if __name__ == "__main__":
     cli()
\ No newline at end of file
