analytics: add crawler tracking, pretty UA/IP display and copy-to-clipboard

frontend/src/AnalyticsView.vue

@@ -8,7 +8,14 @@
 // See docs/analytics.md for the data format.
 import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
 import { RANGES } from './analytics/time.js'
-import { calcTotalViews, formatVisitRows } from './analytics/format.js'
+import {
+  calcTotalViews,
+  copyIp,
+  countCrawlerUas,
+  formatCounts,
+  formatCrawlerRows,
+  formatVisitRows,
+} from './analytics/format.js'
 import * as flagSvgs from 'country-flag-icons/string/3x2'
 import TransitionGraph from './TransitionGraph.vue'
 import VisitorCharts from './VisitorCharts.vue'
@@ -57,6 +64,9 @@ watch(range, (r) => {
 })
 
 const visitRows = computed(() => formatVisitRows(visits.value, pageTree.value))
+const crawlers = computed(() => data.value?.crawlers || [])
+const crawlerRows = computed(() => formatCrawlerRows(crawlers.value))
+const topCrawlerUas = computed(() => countCrawlerUas(crawlers.value).slice(0, 10))
 
 function flagSvg(code) {
   return flagSvgs[code?.toUpperCase()] || ''
@@ -106,7 +116,6 @@ function countryName(code) {
                   <th>trail</th>
                   <th>referer</th>
                   <th>ip</th>
-                  <th>host</th>
                   <th>lang</th>
                   <th>country</th>
                   <th>ua</th>
@@ -123,14 +132,17 @@ function countryName(code) {
                     </a>
                   </td>
                   <td>{{ v.referer }}</td>
-                  <td>{{ v.ip }}</td>
-                  <td>{{ v.host }}</td>
+                  <td>
+                    <span class="clickable-ip"
+                          :title="`Click to copy full IP: ${v.ip}`"
+                          @click="copyIp(v.ip)">{{ v.ipDisplay }}</span>
+                  </td>
                   <td>{{ v.lang }}</td>
                   <td class="country">
                     <span v-if="flagSvg(v.country)" class="flag" v-html="flagSvg(v.country)" :title="countryName(v.country) || v.country"></span>
                     <template v-else>—</template>
                   </td>
-                  <td class="ua" :title="v.ua">{{ v.ua }}</td>
+                  <td class="ua" :title="v.uaRaw">{{ v.ua }}</td>
                   <td>{{ v.utm }}</td>
                 </tr>
               </tbody>
@@ -138,6 +150,42 @@ function countryName(code) {
           </div>
           <p v-else class="empty">no visits recorded yet</p>
         </section>
+
+        <section>
+          <h2>Crawlers</h2>
+          <div v-if="topCrawlerUas.length" class="crawler-top-uas">
+            <p><strong>top UAs:</strong> {{ formatCounts(topCrawlerUas) }}</p>
+          </div>
+          <div v-if="crawlerRows.length" class="visit-table-wrap">
+            <table class="visit-table">
+              <thead>
+                <tr>
+                  <th>when</th>
+                  <th>entry</th>
+                  <th>ip</th>
+                  <th>ua</th>
+                  <th>referer</th>
+                  <th>query</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr v-for="(c, i) in crawlerRows" :key="i">
+                  <td class="when">{{ c.when }}</td>
+                  <td>{{ c.entry }}</td>
+                  <td>
+                    <span class="clickable-ip"
+                          :title="`Click to copy full IP: ${c.ip}`"
+                          @click="copyIp(c.ip)">{{ c.ipDisplay }}</span>
+                  </td>
+                  <td class="ua" :title="c.uaRaw">{{ c.ua }}</td>
+                  <td>{{ c.referer }}</td>
+                  <td>{{ c.query }}</td>
+                </tr>
+              </tbody>
+            </table>
+          </div>
+          <p v-else class="empty">no crawler hits recorded yet</p>
+        </section>
       </template>
     </div>
   </div>
@@ -268,6 +316,16 @@ function countryName(code) {
   margin-left: 0.5rem;
 }
 
+.visit-table .clickable-ip {
+  cursor: pointer;
+  text-decoration: underline;
+  text-decoration-style: dotted;
+}
+
+.visit-table .clickable-ip:hover {
+  color: var(--accent);
+}
+
 .visit-table .ua {
   max-width: 18rem;
   overflow: hidden;
@@ -291,6 +349,15 @@ function countryName(code) {
   display: block;
 }
 
+.crawler-top-uas {
+  font-size: 0.9rem;
+  margin-bottom: 0.6rem;
+}
+
+.crawler-top-uas strong {
+  color: var(--muted);
+}
+
 .empty, .loading, .error { color: var(--muted); }
 .error { color: var(--error, #c00); }
 </style>

frontend/src/analytics/format.js

@@ -3,6 +3,38 @@
  * visit trail.
  */
 
+/**
+ * IPv4 unchanged, IPv6 returns the /64 network prefix in compact form.
+ * Falls back to the original value when parsing fails.
+ */
+export const hostIP = (ip) => {
+  try {
+    if (!ip || !ip.includes(':')) return ip
+    const strip = (s) => s.replace(/^\[|\]$/g, '')
+    const norm = strip(new URL(`http://[${ip}]/`).hostname)
+    const [l, r] = norm.split('::').map((s) => (s ? s.split(':') : []))
+    const full = r
+      ? [...l, ...Array(8 - l.length - r.length).fill('0'), ...r]
+      : l
+    return strip(
+      new URL(`http://[${full.slice(0, 4).join(':')}::]/`).hostname,
+    ).replace(/::$/, '')
+  } catch (e) {
+    console.error('hostIP processing failed for:', ip, e)
+    return ip
+  }
+}
+
+/** Copy the full IP to the clipboard, ignoring failures. */
+export async function copyIp(ip) {
+  if (!ip) return
+  try {
+    await navigator.clipboard.writeText(ip)
+  } catch {
+    /* ignore */
+  }
+}
+
 /** Total page views across every page and every bucket. */
 export function calcTotalViews(views) {
   let n = 0
@@ -87,6 +119,37 @@ export function formatCounts(entries) {
   return entries.map(([value, count]) => `${value} (${count})`).join(', ')
 }
 
+/**
+ * Count distinct User-Agent strings among crawler hits, most common first.
+ * Returns an array of [ua, count] pairs.
+ */
+export function countCrawlerUas(crawlers) {
+  const counts = {}
+  for (const c of crawlers || []) {
+    const value = c.ua_pretty || c.ua || '(no UA)'
+    counts[value] = (counts[value] || 0) + 1
+  }
+  return Object.entries(counts).sort((a, b) => b[1] - a[1])
+}
+
+/**
+ * Format raw crawler hit records as rows for a technical table.  Missing
+ * values become "—".
+ */
+export function formatCrawlerRows(crawlers) {
+  const dash = (s) => (s || '—')
+  return [...(crawlers || [])].reverse().map((c) => ({
+    when: new Date(c.start).toLocaleString(),
+    entry: dash(c.entry),
+    ip: c.ip || '',
+    ipDisplay: c.host || hostIP(c.ip) || c.ip || '—',
+    ua: c.ua_pretty || c.ua || '—',
+    uaRaw: c.ua || '',
+    referer: dash(c.referer),
+    query: dash(c.query),
+  }))
+}
+
 /**
  * Format raw visit records as rows for a technical table.  Returns objects
  * with display strings; missing values become "—".  ``trail`` joins page
@@ -110,11 +173,13 @@ export function formatVisitRows(visits, pageTree) {
       when: new Date(v.start).toLocaleString(),
       trail,
       referer: dash(v.referer),
-      ip: dash(v.ip),
+      ip: v.ip || '',
+      ipDisplay: v.host || hostIP(v.ip) || v.ip || '—',
       host: dash(v.host),
       lang: dash(v.lang),
       country: dash(v.country),
-      ua: dash(v.ua),
+      ua: v.ua_pretty || v.ua || '—',
+      uaRaw: v.ua || '',
       utm: utm || '—',
     }
   })