move json to api, change tile loading api

lib/mound_hunters/application.ex

@@ -48,7 +48,7 @@ defmodule MoundHunters.Application do
       :ok ->
         Logger.info("Created Mnesia schema")
 
-      {:error, {_node1, {:already_exists, _node2}}} ->
+      {:error, {_, {:already_exists, _}}} ->
         Logger.debug("Mnesia schema already exists")
 
       {:error, reason} ->
@@ -74,6 +74,8 @@ defmodule MoundHunters.Application do
              :max_lng,
              :status,
              :error_message,
+             :jpg_available,
+             :png_available,
              :created_at,
              :updated_at
            ],

lib/mound_hunters/mound_parser.ex

@@ -0,0 +1,269 @@
+defmodule MoundHunters.MoundParser do
+  @moduledoc """
+  Parser for .mound binary lidar files.
+
+  Supports:
+  - Header-only parsing
+  - Full file parsing (header + vertices + indices)
+  - Batch header parsing for directories
+  """
+
+  defmodule Header do
+    @moduledoc "Represents a .mound file header"
+
+    defstruct [
+      :magic,
+      :version,
+      :point_count,
+      :triangle_count,
+      :min_x,
+      :min_y,
+      :min_z,
+      :max_x,
+      :max_y,
+      :max_z
+    ]
+
+    @type t :: %__MODULE__{
+            magic: binary(),
+            version: non_neg_integer(),
+            point_count: non_neg_integer(),
+            triangle_count: non_neg_integer(),
+            min_x: float(),
+            min_y: float(),
+            min_z: float(),
+            max_x: float(),
+            max_y: float(),
+            max_z: float()
+          }
+  end
+
+  defmodule MoundFile do
+    @moduledoc "Represents a complete parsed .mound file"
+
+    defstruct [
+      :header,
+      :vertices,
+      :indices
+    ]
+
+    @type vertex :: {float(), float(), float()}
+    @type triangle :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}
+
+    @type t :: %__MODULE__{
+            header: Header.t(),
+            vertices: [vertex()],
+            indices: [triangle()]
+          }
+  end
+
+  @header_size 64
+  @magic "LIDR"
+
+  @doc """
+  Parse only the header of a .mound file.
+
+  ## Examples
+
+      iex> MoundHunters.MoundParser.parse_header("path/to/file.mound")
+      {:ok, %MoundParser.Header{...}}
+      
+      iex> MoundHunters.MoundParser.parse_header("nonexistent.mound")
+      {:error, :enoent}
+  """
+  @spec parse_header(Path.t()) :: {:ok, Header.t()} | {:error, term()}
+  def parse_header(path) do
+    with {:ok, file} <- File.open(path, [:read, :binary]),
+         {:ok, header_data} <- :file.pread(file, 0, @header_size),
+         :ok <- File.close(file),
+         {:ok, header} <- decode_header(header_data) do
+      {:ok, header}
+    end
+  end
+
+  @doc """
+  Parse a complete .mound file including header, vertices, and indices.
+
+  ## Examples
+
+      iex> MoundHunters.MoundParser.parse_file("path/to/file.mound")
+      {:ok, %MoundParser.MoundFile{...}}
+  """
+  @spec parse_file(Path.t()) :: {:ok, MoundFile.t()} | {:error, term()}
+  def parse_file(path) do
+    with {:ok, data} <- File.read(path),
+         {:ok, mound_file} <- decode_file(data) do
+      {:ok, mound_file}
+    end
+  end
+
+  @doc """
+  Parse headers for all .mound files in a directory.
+
+  Returns a list of tuples with filename and header.
+
+  ## Examples
+
+      iex> MoundHunters.MoundParser.parse_directory_headers("tiles/")
+      {:ok, [
+        {"tile_001.mound", %MoundParser.Header{...}},
+        {"tile_002.mound", %MoundParser.Header{...}}
+      ]}
+  """
+  @spec parse_directory_headers(Path.t()) :: {:ok, [{String.t(), Header.t()}]} | {:error, term()}
+  def parse_directory_headers(dir_path) do
+    case File.ls(dir_path) do
+      {:ok, files} ->
+        results =
+          files
+          |> Enum.filter(&String.ends_with?(&1, ".mound"))
+          |> Enum.map(fn filename ->
+            full_path = Path.join(dir_path, filename)
+
+            case parse_header(full_path) do
+              {:ok, header} -> {:ok, {filename, header}}
+              {:error, reason} -> {:error, {filename, reason}}
+            end
+          end)
+
+        # Separate successes from failures
+        {successes, failures} =
+          Enum.split_with(results, fn
+            {:ok, _} -> true
+            {:error, _} -> false
+          end)
+
+        successes = Enum.map(successes, fn {:ok, result} -> result end)
+
+        case failures do
+          [] -> {:ok, successes}
+          _ -> {:ok, successes, Enum.map(failures, fn {:error, err} -> err end)}
+        end
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  # Private functions
+
+  defp decode_header(
+         <<magic::binary-size(4), version::little-unsigned-32, point_count::little-unsigned-32,
+           triangle_count::little-unsigned-32, min_x::little-float-32, min_y::little-float-32,
+           min_z::little-float-32, max_x::little-float-32, max_y::little-float-32,
+           max_z::little-float-32, _reserved::binary-size(24)>>
+       ) do
+    if magic == @magic do
+      {:ok,
+       %Header{
+         magic: magic,
+         version: version,
+         point_count: point_count,
+         triangle_count: triangle_count,
+         min_x: min_x,
+         min_y: min_y,
+         min_z: min_z,
+         max_x: max_x,
+         max_y: max_y,
+         max_z: max_z
+       }}
+    else
+      {:error, {:invalid_magic, magic}}
+    end
+  end
+
+  defp decode_header(_), do: {:error, :invalid_header}
+
+  defp decode_file(data) when byte_size(data) < @header_size do
+    {:error, :file_too_small}
+  end
+
+  defp decode_file(data) do
+    <<header_data::binary-size(@header_size), rest::binary>> = data
+
+    with {:ok, header} <- decode_header(header_data),
+         {:ok, vertices, indices_data} <- decode_vertices(rest, header.point_count),
+         {:ok, indices} <- decode_indices(indices_data, header.triangle_count) do
+      {:ok,
+       %MoundFile{
+         header: header,
+         vertices: vertices,
+         indices: indices
+       }}
+    end
+  end
+
+  defp decode_vertices(data, point_count) do
+    vertex_size = point_count * 12
+    expected_size = point_count * 12
+
+    if byte_size(data) < expected_size do
+      {:error, :insufficient_vertex_data}
+    else
+      <<vertex_data::binary-size(vertex_size), rest::binary>> = data
+      vertices = parse_vertices(vertex_data, point_count, [])
+      {:ok, Enum.reverse(vertices), rest}
+    end
+  end
+
+  defp parse_vertices(<<>>, 0, acc), do: acc
+
+  defp parse_vertices(
+         <<x::little-float-32, y::little-float-32, z::little-float-32, rest::binary>>,
+         count,
+         acc
+       )
+       when count > 0 do
+    parse_vertices(rest, count - 1, [{x, y, z} | acc])
+  end
+
+  defp decode_indices(data, triangle_count) do
+    expected_size = triangle_count * 12
+
+    if byte_size(data) < expected_size do
+      {:error, :insufficient_index_data}
+    else
+      <<index_data::binary-size(expected_size), _rest::binary>> = data
+      indices = parse_indices(index_data, triangle_count, [])
+      {:ok, Enum.reverse(indices)}
+    end
+  end
+
+  defp parse_indices(<<>>, 0, acc), do: acc
+
+  defp parse_indices(
+         <<i1::little-unsigned-32, i2::little-unsigned-32, i3::little-unsigned-32, rest::binary>>,
+         count,
+         acc
+       )
+       when count > 0 do
+    parse_indices(rest, count - 1, [{i1, i2, i3} | acc])
+  end
+
+  @doc """
+  Get file size information without fully parsing.
+
+  Returns expected file size based on header counts.
+  """
+  @spec expected_file_size(Header.t()) :: non_neg_integer()
+  def expected_file_size(%Header{point_count: pc, triangle_count: tc}) do
+    @header_size + pc * 12 + tc * 12
+  end
+
+  @doc """
+  Validate that a file has the correct size based on its header.
+  """
+  @spec validate_file_size(Path.t()) :: {:ok, :valid} | {:error, term()}
+  def validate_file_size(path) do
+    with {:ok, %{size: actual_size}} <- File.stat(path),
+         {:ok, header} <- parse_header(path) do
+      expected = expected_file_size(header)
+
+      if actual_size == expected do
+        {:ok, :valid}
+      else
+        {:error, {:size_mismatch, expected: expected, actual: actual_size}}
+      end
+    end
+  end
+end

lib/mound_hunters/repo.ex

@@ -9,15 +9,52 @@ defmodule MoundHunters.Repo do
   @type tile_status :: :processing | :ready | :error
 
   @doc """
-  Get a tile by ID from Mnesia.
+  Get a tile by its ID.
+  Returns {:ok, tile} or {:error, :not_found}
   """
-  def get_tile(tile_id) do
+  def get_tile(id) do
     case :mnesia.transaction(fn ->
-           :mnesia.read(:tiles, tile_id)
+           :mnesia.read(:tiles, id)
          end) do
-      {:atomic, [tile]} -> {:ok, tile_to_map(tile)}
-      {:atomic, []} -> {:error, :not_found}
-      {:aborted, reason} -> {:error, reason}
+      {:atomic, [tile]} ->
+        {:ok, tile_to_map(tile)}
+
+      {:atomic, []} ->
+        {:error, :not_found}
+
+      {:aborted, reason} ->
+        Logger.error("Failed to read tile #{id}: #{inspect(reason)}")
+        {:error, :database_error}
+    end
+  end
+
+  @doc """
+  Get a tile that contains the given lat/lng coordinates.
+  Returns {:ok, tile} or {:error, :not_found}
+
+  If multiple tiles contain the point, returns the first match.
+  """
+  def get_tile_at_coords(lat, lng) do
+    match_spec = [
+      {{:tiles, :"$1", :"$2", :"$3", :"$4", :"$5", :"$6", :"$7", :"$8", :"$9", :"$10", :"$11"},
+       [
+         {:andalso, {:"=<", :"$2", lat}, {:"=<", lat, :"$3"}},
+         {:andalso, {:"=<", :"$4", lng}, {:"=<", lng, :"$5"}}
+       ], [:"$_"]}
+    ]
+
+    case :mnesia.transaction(fn ->
+           :mnesia.select(:tiles, match_spec)
+         end) do
+      {:atomic, [tile | _]} ->
+        {:ok, tile_to_map(tile)}
+
+      {:atomic, []} ->
+        {:error, :not_found}
+
+      {:aborted, reason} ->
+        Logger.error("Failed to query tiles at #{lat}, #{lng}: #{inspect(reason)}")
+        {:error, :database_error}
     end
   end
 
@@ -31,7 +68,8 @@ defmodule MoundHunters.Repo do
     tile_record =
       {:tiles, tile_id, Map.get(attrs, :min_lat), Map.get(attrs, :max_lat),
        Map.get(attrs, :min_lng), Map.get(attrs, :max_lng), Map.get(attrs, :status),
-       Map.get(attrs, :error_message), Map.get(attrs, :created_at, now), now}
+       Map.get(attrs, :error_message), Map.get(attrs, :jpg_available, false),
+       Map.get(attrs, :png_available, false), Map.get(attrs, :created_at, now), now}
 
     case :mnesia.transaction(fn ->
            :mnesia.write(tile_record)
@@ -73,8 +111,8 @@ defmodule MoundHunters.Repo do
 
   # Convert Mnesia tile record to map
   defp tile_to_map(
-         {:tiles, id, min_lat, max_lat, min_lng, max_lng, status, error_message, created_at,
-          updated_at}
+         {:tiles, id, min_lat, max_lat, min_lng, max_lng, status, error_message, jpg_available,
+          png_available, created_at, updated_at}
        ) do
     %{
       id: id,
@@ -84,6 +122,8 @@ defmodule MoundHunters.Repo do
       max_lng: max_lng,
       status: status,
       error_message: error_message,
+      jpg_available: jpg_available,
+      png_available: png_available,
       created_at: created_at,
       updated_at: updated_at
     }
@@ -98,6 +138,175 @@ defmodule MoundHunters.Repo do
     }
   end
 
+  @doc """
+  Repopulate tiles table from .mound files in the configured directory.
+  Skips tiles that already exist in the database.
+  Returns {success_count, skip_count, error_count}.
+  """
+  def repopulate_from_mounds do
+    tile_dir = Application.get_env(:mound_hunters, :tile_output_dir)
+    mound_dir = Path.join(tile_dir, "MOUND")
+
+    Logger.info("Scanning for .mound files in: #{mound_dir}")
+
+    case File.ls(mound_dir) do
+      {:ok, files} ->
+        results =
+          files
+          |> Enum.filter(&String.ends_with?(&1, ".mound"))
+          |> Enum.map(fn filename ->
+            # Extract tile_id from filename (e.g., "BS18921654.mound" -> "BS18921654")
+            tile_id = Path.rootname(filename)
+            full_path = Path.join(mound_dir, filename)
+
+            process_mound_file(tile_id, full_path)
+          end)
+
+        successes = Enum.count(results, &(&1 == :ok))
+        skips = Enum.count(results, &(&1 == :skipped))
+        errors = Enum.count(results, &match?({:error, _}, &1))
+
+        Logger.info(
+          "Repopulation complete: #{successes} inserted, #{skips} skipped, #{errors} errors"
+        )
+
+        {successes, skips, errors}
+
+      {:error, reason} ->
+        Logger.error("Failed to list directory #{mound_dir}: #{inspect(reason)}")
+        {:error, reason}
+    end
+  end
+
+  @doc """
+  Scan tile directories and update jpg_available and png_available flags.
+  Returns {updated_count, not_found_count}.
+  """
+  def update_tile_availability do
+    tile_dir = Application.get_env(:mound_hunters, :tile_output_dir)
+    mound_dir = Path.join(tile_dir, "MOUND")
+
+    Logger.info("Scanning tile availability in: #{mound_dir}")
+
+    # Get all tile IDs from database
+    case :mnesia.transaction(fn ->
+           :mnesia.match_object({:tiles, :_, :_, :_, :_, :_, :_, :_, :_, :_, :_, :_})
+         end) do
+      {:atomic, tiles} ->
+        results =
+          tiles
+          |> Enum.map(fn tile_record ->
+            tile = tile_to_map(tile_record)
+            tile_id = tile.id
+
+            jpg_path = Path.join(mound_dir, "#{tile_id}.jpg")
+            png_path = Path.join(mound_dir, "#{tile_id}.png")
+
+            jpg_available = File.exists?(jpg_path)
+            png_available = File.exists?(png_path)
+
+            # Only update if availability changed
+            if jpg_available != tile.jpg_available or png_available != tile.png_available do
+              attrs = %{
+                id: tile_id,
+                min_lat: tile.min_lat,
+                max_lat: tile.max_lat,
+                min_lng: tile.min_lng,
+                max_lng: tile.max_lng,
+                status: tile.status,
+                error_message: tile.error_message,
+                jpg_available: jpg_available,
+                png_available: png_available,
+                created_at: tile.created_at
+              }
+
+              case upsert_tile(attrs) do
+                {:ok, _} ->
+                  Logger.debug("Updated #{tile_id}: jpg=#{jpg_available}, png=#{png_available}")
+
+                  :updated
+
+                {:error, reason} ->
+                  Logger.error("Failed to update #{tile_id}: #{inspect(reason)}")
+                  {:error, reason}
+              end
+            else
+              :unchanged
+            end
+          end)
+
+        updated = Enum.count(results, &(&1 == :updated))
+        unchanged = Enum.count(results, &(&1 == :unchanged))
+        errors = Enum.count(results, &match?({:error, _}, &1))
+
+        Logger.info(
+          "Availability scan complete: #{updated} updated, #{unchanged} unchanged, #{errors} errors"
+        )
+
+        {updated, unchanged, errors}
+
+      {:aborted, reason} ->
+        Logger.error("Failed to read tiles: #{inspect(reason)}")
+        {:error, reason}
+    end
+  end
+
+  defp process_mound_file(tile_id, file_path) do
+    # Check if tile already exists
+    case get_tile(tile_id) do
+      {:ok, _existing} ->
+        Logger.debug("Skipping existing tile: #{tile_id}")
+        :skipped
+
+      {:error, :not_found} ->
+        # Parse header and insert
+        case MoundHunters.MoundParser.parse_header(file_path) do
+          {:ok, header} ->
+            # Convert Web Mercator coordinates to lat/lng
+            min_lng = header.min_x
+            max_lng = header.max_x
+            min_lat = header.min_y
+            max_lat = header.max_y
+
+            # Check for jpg and png availability
+            tile_dir = Application.get_env(:mound_hunters, :tile_output_dir)
+            mound_dir = Path.join(tile_dir, "MOUND")
+            jpg_available = File.exists?(Path.join(mound_dir, "#{tile_id}.jpg"))
+            png_available = File.exists?(Path.join(mound_dir, "#{tile_id}.png"))
+
+            attrs = %{
+              id: tile_id,
+              min_lat: min_lat,
+              max_lat: max_lat,
+              min_lng: min_lng,
+              max_lng: max_lng,
+              status: :ready,
+              error_message: nil,
+              jpg_available: jpg_available,
+              png_available: png_available
+            }
+
+            case upsert_tile(attrs) do
+              {:ok, _tile} ->
+                Logger.debug("Inserted tile: #{tile_id}")
+                :ok
+
+              {:error, reason} ->
+                Logger.error("Failed to insert tile #{tile_id}: #{inspect(reason)}")
+                {:error, reason}
+            end
+
+          {:error, reason} ->
+            Logger.error("Failed to parse header for #{tile_id}: #{inspect(reason)}")
+            {:error, reason}
+        end
+
+      {:error, reason} ->
+        Logger.error("Failed to check if tile exists #{tile_id}: #{inspect(reason)}")
+        {:error, reason}
+    end
+  end
+
   # Generate random 8-character alphanumeric ID
   defp generate_id do
     :crypto.strong_rand_bytes(6)