From ad1c093199233e2057e84481726c78e5065424d8 Mon Sep 17 00:00:00 2001 From: Eric Downing Date: Thu, 24 Sep 2026 21:13:19 -0700 Subject: [PATCH] moving to mysql --- .env.example | 12 +- .env.testing.example | 65 ++++++ .github/workflows/tests.yml | 23 ++ .gitignore | 1 + app/Console/Commands/TransferSqliteData.php | 140 +++++++++++ config/database.php | 10 +- migration-guide.md | 247 ++++++++++++++++++++ phpunit.xml | 7 +- 8 files changed, 495 insertions(+), 10 deletions(-) create mode 100644 .env.testing.example create mode 100644 app/Console/Commands/TransferSqliteData.php create mode 100644 migration-guide.md diff --git a/.env.example b/.env.example index a2cc856..36ccc0c 100644 --- a/.env.example +++ b/.env.example @@ -24,12 +24,12 @@ LOG_STACK=single LOG_DEPRECATIONS_CHANNEL=null LOG_LEVEL=debug -DB_CONNECTION=sqlite -# DB_HOST=127.0.0.1 -# DB_PORT=3306 -# DB_DATABASE=laravel -# DB_USERNAME=root -# DB_PASSWORD= +DB_CONNECTION=mysql +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DATABASE=mytodos +DB_USERNAME=root +DB_PASSWORD= SESSION_DRIVER=database SESSION_LIFETIME=120 diff --git a/.env.testing.example b/.env.testing.example new file mode 100644 index 0000000..91be2c5 --- /dev/null +++ b/.env.testing.example @@ -0,0 +1,65 @@ +APP_NAME=Laravel +APP_ENV=testing +APP_KEY= +APP_DEBUG=true +APP_URL=http://localhost:8000 + +APP_LOCALE=en +APP_FALLBACK_LOCALE=en +APP_FAKER_LOCALE=en_US + +APP_MAINTENANCE_DRIVER=file +# APP_MAINTENANCE_STORE=database + +# PHP_CLI_SERVER_WORKERS=4 + +BCRYPT_ROUNDS=4 + +LOG_CHANNEL=stack +LOG_STACK=single +LOG_DEPRECATIONS_CHANNEL=null +LOG_LEVEL=debug + +DB_CONNECTION=mysql +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DATABASE=mytodos_test +DB_USERNAME=root +DB_PASSWORD= + +SESSION_DRIVER=array +SESSION_LIFETIME=120 +SESSION_ENCRYPT=false +SESSION_PATH=/ +SESSION_DOMAIN=null + +BROADCAST_CONNECTION=log +FILESYSTEM_DISK=local +QUEUE_CONNECTION=sync + +CACHE_STORE=array +# CACHE_PREFIX= + +MEMCACHED_HOST=127.0.0.1 + +REDIS_CLIENT=phpredis +REDIS_HOST=127.0.0.1 +REDIS_PASSWORD=null +REDIS_PORT=6379 + +MAIL_MAILER=array +MAIL_SCHEME=null +MAIL_HOST=127.0.0.1 +MAIL_PORT=2525 +MAIL_USERNAME=null +MAIL_PASSWORD=null +MAIL_FROM_ADDRESS="hello@example.com" +MAIL_FROM_NAME="${APP_NAME}" + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_DEFAULT_REGION=us-east-1 +AWS_BUCKET= +AWS_USE_PATH_STYLE_ENDPOINT=false + +VITE_APP_NAME="${APP_NAME}" \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 86de102..c05adc8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,6 +13,29 @@ jobs: ci: runs-on: ubuntu-latest + env: + DB_CONNECTION: mysql + DB_HOST: 127.0.0.1 + DB_PORT: 3306 + DB_DATABASE: mytodos_test + DB_USERNAME: root + DB_PASSWORD: password + + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: mytodos_test + MYSQL_ROOT_HOST: '%' + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=10 + steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.gitignore b/.gitignore index 6c91fb8..adf9fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .env .env.backup .env.production +.env.testing .phpactor.json .phpunit.result.cache Homestead.json diff --git a/app/Console/Commands/TransferSqliteData.php b/app/Console/Commands/TransferSqliteData.php new file mode 100644 index 0000000..fbd7442 --- /dev/null +++ b/app/Console/Commands/TransferSqliteData.php @@ -0,0 +1,140 @@ + + */ + private const TABLES = [ + 'users' => 'id', + 'folders' => 'id', + 'todos' => 'id', + 'passkeys' => 'id', + 'password_reset_tokens' => 'email', + ]; + + /** + * Execute the console command. + */ + public function handle(): int + { + if ($this->option('source') !== null) { + config([ + 'database.connections.sqlite_source.database' => $this->option('source'), + ]); + DB::purge('sqlite_source'); + } + + $source = (string) config('database.connections.sqlite_source.database'); + + if (! is_file($source)) { + $this->error("Source database not found: {$source}"); + + return self::FAILURE; + } + + if (DB::connection()->getDriverName() !== 'mysql') { + $this->error('The destination connection must be MySQL.'); + + return self::FAILURE; + } + + if (app()->environment('production') && ! $this->option('force')) { + $this->error('Refusing to run in production without --force.'); + + return self::FAILURE; + } + + if (! $this->option('force') && ! $this->confirm( + 'This truncates and replaces the domain tables in the destination database. Continue?' + )) { + $this->info('Aborted.'); + + return self::FAILURE; + } + + $this->info("Copying data from {$source} into the 'mysql' connection."); + + try { + Schema::disableForeignKeyConstraints(); + + foreach (self::TABLES as $table => $key) { + $this->transferTable($table, $key); + } + + $this->resetAutoIncrements(); + } catch (Throwable $e) { + $this->error('Transfer failed: '.$e->getMessage()); + + return self::FAILURE; + } finally { + Schema::enableForeignKeyConstraints(); + } + + $this->info('Data transfer complete.'); + + return self::SUCCESS; + } + + /** + * Copy the contents of one table from the source into the destination. + */ + private function transferTable(string $table, string $key): void + { + if (! Schema::connection('sqlite_source')->hasTable($table)) { + $this->line(" skipped {$table}: not present in the source database."); + + return; + } + + if (! Schema::hasTable($table)) { + $this->line(" skipped {$table}: not present in the destination database."); + + return; + } + + $count = (int) DB::connection('sqlite_source')->table($table)->count(); + + DB::table($table)->truncate(); + + DB::connection('sqlite_source') + ->table($table) + ->orderBy($key) + ->chunk(500, function ($rows) use ($table): void { + DB::table($table)->insert($rows->map(fn ($row): array => (array) $row)->all()); + }); + + $this->line(" transferred {$count} row(s) into {$table}."); + } + + /** + * Bump each auto-increment counter past the highest transferred id so + * newly created records never collide with the preserved ids. + */ + private function resetAutoIncrements(): void + { + foreach (['users', 'folders', 'todos', 'passkeys'] as $table) { + $max = DB::table($table)->max('id'); + + if ($max !== null) { + DB::statement("ALTER TABLE `{$table}` AUTO_INCREMENT = ".((int) $max + 1)); + } + } + } +} diff --git a/config/database.php b/config/database.php index abbb88e..78eadf3 100644 --- a/config/database.php +++ b/config/database.php @@ -17,7 +17,7 @@ return [ | */ - 'default' => env('DB_CONNECTION', 'sqlite'), + 'default' => env('DB_CONNECTION', 'mysql'), /* |-------------------------------------------------------------------------- @@ -44,6 +44,14 @@ return [ 'transaction_mode' => 'DEFERRED', ], + 'sqlite_source' => [ + 'driver' => 'sqlite', + 'url' => env('SQLITE_SOURCE_URL'), + 'database' => env('SQLITE_SOURCE_DATABASE', database_path('database.sqlite')), + 'prefix' => '', + 'foreign_key_constraints' => false, + ], + 'mysql' => [ 'driver' => 'mysql', 'url' => env('DB_URL'), diff --git a/migration-guide.md b/migration-guide.md new file mode 100644 index 0000000..cb63fda --- /dev/null +++ b/migration-guide.md @@ -0,0 +1,247 @@ +# Production Migration Guide: SQLite → MySQL + +This runbook moves the application's production database from SQLite to MySQL 8, +preserving existing data. It mirrors the local cutover and relies on the +`db:transfer-sqlite` command added in `app/Console/Commands/TransferSqliteData.php`. + +Read the whole guide before starting. Every step assumes you can SSH into the +production server and run commands from the app root. + +--- + +## 0. Preconditions + +- MySQL 8 server reachable from the app server (5.7+ and MariaDB also work, but MySQL 8 is the tested target). +- A database user with privileges to create the database and tables. +- **Put the app into read-only/maintenance mode during the cutover** so no new + writes land in SQLite while you migrate: + +```bash +php artisan down +``` + +- Record the size and location of the current database: + +```bash +ls -lh database/database.sqlite +``` + +Note the path. If it is not `database/database.sqlite`, you will pass it to the +transfer command with `--source`. + +--- + +## 1. Back up the SQLite database + +Keep the SQLite file intact all the way through. Copy it to a timestamped backup +and move the original aside (it is the documented rollback path): + +```bash +cp database/database.sqlite database/database.sqlite.bak-$(date +%Y%m%d%H%M%S) +``` + +The transfer command reads the file directly, so the original must still be at +`database/database.sqlite` (or referenced via `--source`) during the cutover. + +--- + +## 2. Create the MySQL database and user + +Run from a MySQL client with admin access: + +```sql +CREATE DATABASE mytodos + CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +CREATE USER 'mytodos'@'%' IDENTIFIED BY ''; +GRANT ALL PRIVILEGES ON mytodos.* TO 'mytodos'@'%'; +FLUSH PRIVILEGES; +``` + +Use a user scoped to `mytodos.*` — **do not grant global privileges**. Restrict +`'mytodos'@'localhost'` instead of `'%'` if the app always connects via the local +socket/host. + +--- + +## 3. Deploy the MySQL-compatible code + +The code changes from the local migration must be on the server: + +```bash +git pull --ff-only origin main +composer install --no-dev --prefer-dist --optimize-autoloader +npm ci --no-audit --no-fund +npm run build +``` + +Relevant changes in this release: +- `config/database.php` — MySQL default connection + `sqlite_source` connection. +- `app/Console/Commands/TransferSqliteData.php` — the `db:transfer-sqlite` command. +- `phpunit.xml`, `.env.testing(.example)`, `.github/workflows/tests.yml` — test/CI only, no production impact. + +--- + +## 4. Update the environment + +Switch the application to MySQL, using the credentials from step 2: + +```dotenv +DB_CONNECTION=mysql +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DATABASE=mytodos +DB_USERNAME=mytodos +DB_PASSWORD= +``` + +- Keep the old `database/database.sqlite` file in place (see step 6). +- If you previously ran `php artisan config:cache`, the config is cached and will + not pick up the new `.env` until step 8. Run `php artisan config:clear` before + the next `config:cache`, or rely on the cache rebuild in step 8. + +--- + +## 5. Run the migrations against the empty MySQL database + +The MySQL database is still empty. Apply the schema: + +```bash +php artisan migrate --force +``` + +Do not seed yet — the data comes from SQLite in the next step. + +--- + +## 6. Transfer the data + +Run the transfer command. It copies the domain tables (`users`, `folders`, +`todos`, `passkeys`, `password_reset_tokens`) from SQLite into MySQL, preserving +IDs, then resets MySQL auto-increment counters. + +If the SQLite file is at the default path: + +```bash +php artisan db:transfer-sqlite --force +``` + +If it lives elsewhere: + +```bash +php artisan db:transfer-sqlite --source=/var/www/app/database/database.sqlite --force +``` + +The `--force` flag is required because `APP_ENV` is `production`. The command: + +- aborts unless the destination connection is MySQL; +- truncates the destination domain tables before inserting (it is idempotent); +- skips framework/transient tables (`cache`, `sessions`, `jobs`, `migrations`, …). + +Expected output: + +``` +Copying data from /var/www/app/database/database.sqlite into the 'mysql' connection. + transferred N row(s) into users. + transferred N row(s) into folders. + ... +Data transfer complete. +``` + +--- + +## 7. Verify the transfer + +Compare row counts and spot-check a sample: + +```bash +php artisan tinker --execute ' + $source = DB::connection("sqlite_source"); + foreach (["users", "folders", "todos", "passkeys"] as $table) { + echo $table.": ".$source->table($table)->count()." -> ".DB::table($table)->count().PHP_EOL; + } +' +``` + +Manually confirm a few rows and the max IDs on MySQL: + +```bash +php artisan tinker --execute ' + foreach (["users", "folders", "todos", "passkeys"] as $table) { + echo $table." max id: ".DB::table($table)->max("id").PHP_EOL; + } +' +``` + +Then re-create the admin/administrator account that `BaseDatabaseSeeder` owns: + +```bash +php artisan db:seed --class=BaseDatabaseSeeder --force +``` + +--- + +## 8. Finalize + +```bash +php artisan config:cache +php artisan route:cache +php artisan view:cache +php artisan queue:restart +php artisan up +``` + +Smoke-test from outside: log in, load the dashboard, run a search, toggle a +status filter, and drag-reorder a todo. + +--- + +## 9. Post-migration cleanup (optional, after a soak period) + +Once you are confident MySQL is healthy: + +- Archive the SQLite backup off the server (or delete it) after you have confirmed + there are no rollback triggers. +- Consider deleting the one-time `app/Console/Commands/TransferSqliteData.php` + command so it can never truncate production tables by accident. +- Update `deploy.sh`'s comments if they reference SQLite. + +--- + +## Rollback + +1. `php artisan down`. +2. Point the app back at SQLite: + +```dotenv +DB_CONNECTION=sqlite +# remove the DB_HOST/DB_PORT/DB_DATABASE/DB_USERNAME/DB_PASSWORD MySQL lines +``` + +3. Replace the (unmigrated) original with your backup: + +```bash +cp database/database.sqlite.bak- database/database.sqlite +``` + +4. `php artisan config:clear && php artisan up`. + +Note: any rows written into MySQL after step 5 are not on the SQLite backup and +will be lost on rollback. + +--- + +## Production notes + +- **Ordering matters**: migrate → transfer → seed. Running `db:seed` before the + transfer would create a conflicting admin user truncated by the transfer's + destination-table wipe. +- **Timezone**: SQLite stores Laravel timestamps as `YYYY-MM-DD HH:MM:SS` text. + Those strings transfer verbatim and MySQL interprets them as-is; ensure the + MySQL session timezone matches the app's `app.timezone`/`APP_TIMEZONE` if you + observe hour shifts. +- **Search behavior**: `LIKE` becomes case- and accent-insensitive under + `utf8mb4_unicode_ci` — a deliberate, generally desirable change from SQLite's + ASCII-only behavior. +- **Index/key limits**: the schema requires MySQL 5.7.7+ / 8.0 (InnoDB 3072-byte + key limit). Do not run this against MySQL 5.6. \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index e7f0a48..0b0adaf 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -23,9 +23,10 @@ - - - + + + +