From 6dd1550724a1f6a3489f73e753e5674e840a3698 Mon Sep 17 00:00:00 2001
From: Aaron Parecki <aaron@parecki.com>
Date: Tue, 22 Aug 2017 08:21:40 -0700
Subject: [PATCH] add task to recreate file without duplicate data

---
 compass/app/Console/Commands/CleanFile.php | 45 ++++++++++++++++++++++
 compass/app/Console/Kernel.php             |  1 +
 2 files changed, 46 insertions(+)
 create mode 100644 compass/app/Console/Commands/CleanFile.php

diff --git a/compass/app/Console/Commands/CleanFile.php b/compass/app/Console/Commands/CleanFile.php
new file mode 100644
index 0000000..ff5bdcc
--- /dev/null
+++ b/compass/app/Console/Commands/CleanFile.php
@@ -0,0 +1,45 @@
+<?php
+namespace App\Console\Commands;
+use Illuminate\Console\Command;
+use Laravel\Lumen\Routing\DispatchesJobs;
+use Log;
+use DateTime, DateTimeZone;
+
+class Cleanfile extends Command {
+
+  protected $signature = 'clean:file {in} {out}';
+  protected $description = 'Discard duplicate out of order data from a file';
+
+  public function handle() {
+    $in = $this->argument('in');
+    $out = $this->argument('out');
+
+    $fp = @fopen($in, 'r');
+    $outf = @fopen($out, 'w');
+    if($fp && $outf) {
+
+      $last = false;
+      while(($line = fgets($fp)) !== false) {
+        $cur = new DateTime(substr($line, 0, 26), new DateTimeZone('UTC'));
+
+        if(!$last) {
+          $last = new DateTime(substr($line, 0, 26), new DateTimeZone('UTC'));
+          fwrite($outf, $line);
+        } else {
+          if((double)$cur->format('U.u') > (double)$last->format('U.u')) {
+            fwrite($outf, $line);
+            $last = new DateTime(substr($line, 0, 26), new DateTimeZone('UTC'));
+          } else {
+            Log::info("Discarding line");
+          }
+        }
+      }
+      fclose($fp);
+      fclose($outf);
+
+    } else {
+      Log::error("Could not find input file");
+    }
+  }
+
+}
diff --git a/compass/app/Console/Kernel.php b/compass/app/Console/Kernel.php
index 876ca5b..8d20019 100644
--- a/compass/app/Console/Kernel.php
+++ b/compass/app/Console/Kernel.php
@@ -14,6 +14,7 @@ class Kernel extends ConsoleKernel
      */
     protected $commands = [
       'App\Console\Commands\TestTrip',
+      'App\Console\Commands\CleanFile',
     ];
 
     /**