数据统计中心计划任务脚本，页面，接口

addons/shopro/job/TrackProcessJob.php

@@ -7,6 +7,10 @@ use think\queue\Job;
 use think\Db;
 use think\Queue;
 
+/**
+ * 埋点数据处理任务 
+ * PV UV 等统计
+ */
 class TrackProcessJob
 {
     /**
@@ -16,6 +20,7 @@ class TrackProcessJob
         try {
             $trackData = $data['data'] ?? [];
             $clientIp = $data['ip'] ?? '';
+            $jobTimestamp = isset($data['timestamp']) ? (int)$data['timestamp'] : time();
 
             if (empty($trackData)) {
                 $job->delete();
@@ -23,7 +28,7 @@ class TrackProcessJob
             }
 
             // 批量插入埋点数据
-            $this->insertTracks($trackData, $clientIp);
+            $this->insertTracks($trackData, $clientIp, $jobTimestamp);
 
             //  删除任务
             $job->delete();
@@ -43,22 +48,39 @@ class TrackProcessJob
     /**
      * 批量插入埋点数据
      */
-    protected function insertTracks($trackData, $clientIp) {
+    protected function insertTracks($trackData, $clientIp, $jobTimestamp) {
         $insertData = [];
-        $now = time();
+        $seen = [];
 
         foreach ($trackData as $item) {
-            $insertData[] = [
+            $trackRow = [
                 'user_id' => $item['user_id'] ?? 0,
                 'openid' => $item['openid'] ?? '',
                 'session_id' => $item['session_id'] ?? '',
                 'event_type' => $item['event_type'] ?? '',
+                'event_code' => $item['event_code'] ?? '',
                 'page_path' => $item['page_path'] ?? '',
                 'page_title' => $item['page_title'] ?? '',
                 'params' => !empty($item['params']) ? json_encode($item['params']) : '',
                 'ip' => $clientIp,
-                'create_time' => isset($item['create_time']) ? (int)($item['create_time'] / 1000) : $now
+                'create_time' => isset($item['create_time']) ? (int)($item['create_time'] / 1000) : $jobTimestamp
             ];
+
+            $dedupKey = $this->buildTrackDedupKey($trackRow);
+            if (isset($seen[$dedupKey])) {
+                continue;
+            }
+            $seen[$dedupKey] = true;
+
+            if ($this->trackExists($trackRow)) {
+                continue;
+            }
+
+            $insertData[] = $trackRow;
+        }
+
+        if (empty($insertData)) {
+            return;
         }
 
         // 分批插入（每批500条）
@@ -69,10 +91,48 @@ class TrackProcessJob
     }
 
     /**
+     * 构建埋点判重键
+     */
+    protected function buildTrackDedupKey($trackRow)
+    {
+        return md5(implode('|', [
+            (string)$trackRow['user_id'],
+            (string)$trackRow['openid'],
+            (string)$trackRow['session_id'],
+            (string)$trackRow['event_type'],
+            (string)$trackRow['event_code'],
+            (string)$trackRow['page_path'],
+            (string)$trackRow['page_title'],
+            (string)$trackRow['params'],
+            (string)$trackRow['ip'],
+            (string)$trackRow['create_time'],
+        ]));
+    }
+
+    /**
+     * 判断埋点是否已存在
+     */
+    protected function trackExists($trackRow)
+    {
+        return Db::name('shopro_miniprogram_track')
+            ->where('user_id', $trackRow['user_id'])
+            ->where('openid', $trackRow['openid'])
+            ->where('session_id', $trackRow['session_id'])
+            ->where('event_type', $trackRow['event_type'])
+            ->where('event_code', $trackRow['event_code'])
+            ->where('page_path', $trackRow['page_path'])
+            ->where('page_title', $trackRow['page_title'])
+            ->where('params', $trackRow['params'])
+            ->where('ip', $trackRow['ip'])
+            ->where('create_time', $trackRow['create_time'])
+            ->find() ? true : false;
+    }
+
+    /**
      * 保存失败数据
      */
     protected function saveFailed($data, $error) {
-        Db::name('shopro_track_failed')->insert([
+        Db::name('shopro_miniprogram_track_failed')->insert([
             'data' => json_encode($data),
             'error' => $error,
             'attempts' => 3,
@@ -81,4 +141,4 @@ class TrackProcessJob
     }
 
 
-}
+}
\ No newline at end of file

application/admin/view/shopro/common/script.html

 <link rel="stylesheet" href="//at.alicdn.com/t/c/font_2385137_b8qygb2jne.css?v={$site.version|htmlentities}">
 <link rel="stylesheet" href="__CDN__/assets/addons/shopro/css/index.css?v={$site.version|htmlentities}">
+
+<script src="__CDN__/assets/libs/jquery/dist/jquery.min.js?v={$site.version|htmlentities}"></script>
 {if condition="$DARK_TYPE neq 'none'" }
 <link rel="stylesheet" href="__CDN__/assets/addons/shopro/css/dark.css?v={$site.version|htmlentities}" data-render="darktheme" />
 {/if}

application/command.php

@@ -10,6 +10,8 @@
 // | Author: yunwuxin <448901948@qq.com>
 // +----------------------------------------------------------------------
 
+use Dom\DtdNamedNodeMap;
+
 return [
     'app\admin\command\Crud',
     'app\admin\command\Menu',
@@ -21,4 +23,6 @@ return [
     'app\command\StatMemberTag',
     'app\command\StatsDaily',
     'app\command\TrackFailedRetry',
+    'app\command\DataStatsCenter',
+
 ];

application/command/TrackFailedRetry.php

@@ -36,6 +36,7 @@ class TrackFailedRetry extends Command
     }
 
     protected function execute(Input $input, Output $output) {
+        $this->output = $output;
 
         $this->customLog('info', '========== [' . date('Y-m-d H:i:s') . '] 开始执行重试失败埋埋点数据任务 ==========');
 
@@ -78,8 +79,24 @@ class TrackFailedRetry extends Command
                         throw new \Exception('数据格式错误');
                     }
 
+                    $retryPayload = $this->filterRetryPayload($data);
+
+                    if (empty($retryPayload['data'])) {
+                        Db::name('shopro_miniprogram_track_failed')
+                            ->where('id', $item['id'])
+                            ->update([
+                                'status' => 1,
+                                'error' => '同一操作已入库，跳过重试',
+                                'update_time' => time()
+                            ]);
+
+                        $totalSuccess++;
+                        $totalProcessed++;
+                        continue;
+                    }
+
                     // 重新推入队列
-                    Queue::push('\addons\shopro\job\TrackProcessJob@fire', $data['data'], 'shopro-high');
+                    Queue::push('\addons\shopro\job\TrackProcessJob@fire', $retryPayload, 'shopro-high');
 
                     // 更新状态为已处理
                     Db::name('shopro_miniprogram_track_failed')
@@ -114,6 +131,94 @@ class TrackFailedRetry extends Command
         $this->customLog('info',  "重试完成！总计处理: {$totalProcessed} | 成功: {$totalSuccess} | 失败: {$totalFailed}");
     }
 
+    /**
+     * 过滤已经入库的埋点，仅重试未入库数据
+     */
+    protected function filterRetryPayload($payload)
+    {
+        $trackList = isset($payload['data']) && is_array($payload['data']) ? $payload['data'] : [];
+        $clientIp = $payload['ip'] ?? '';
+        $jobTimestamp = isset($payload['timestamp']) ? (int)$payload['timestamp'] : time();
+        $pendingList = [];
+        $seen = [];
+
+        foreach ($trackList as $trackItem) {
+            $trackRow = $this->buildTrackRow($trackItem, $clientIp, $jobTimestamp);
+            $dedupKey = $this->buildTrackDedupKey($trackRow);
+
+            if (isset($seen[$dedupKey])) {
+                continue;
+            }
+            $seen[$dedupKey] = true;
+
+            if ($this->trackExists($trackRow)) {
+                continue;
+            }
+
+            $pendingList[] = $trackItem;
+        }
+
+        $payload['data'] = $pendingList;
+        return $payload;
+    }
+
+    /**
+     * 构建埋点行数据
+     */
+    protected function buildTrackRow($trackItem, $clientIp, $jobTimestamp)
+    {
+        return [
+            'user_id' => $trackItem['user_id'] ?? 0,
+            'openid' => $trackItem['openid'] ?? '',
+            'session_id' => $trackItem['session_id'] ?? '',
+            'event_type' => $trackItem['event_type'] ?? '',
+            'event_code' => $trackItem['event_code'] ?? '',
+            'page_path' => $trackItem['page_path'] ?? '',
+            'page_title' => $trackItem['page_title'] ?? '',
+            'params' => !empty($trackItem['params']) ? json_encode($trackItem['params']) : '',
+            'ip' => $clientIp,
+            'create_time' => isset($trackItem['create_time']) ? (int)($trackItem['create_time'] / 1000) : $jobTimestamp
+        ];
+    }
+
+    /**
+     * 构建埋点判重键
+     */
+    protected function buildTrackDedupKey($trackRow)
+    {
+        return md5(implode('|', [
+            (string)$trackRow['user_id'],
+            (string)$trackRow['openid'],
+            (string)$trackRow['session_id'],
+            (string)$trackRow['event_type'],
+            (string)$trackRow['event_code'],
+            (string)$trackRow['page_path'],
+            (string)$trackRow['page_title'],
+            (string)$trackRow['params'],
+            (string)$trackRow['ip'],
+            (string)$trackRow['create_time'],
+        ]));
+    }
+
+    /**
+     * 判断埋点是否已存在
+     */
+    protected function trackExists($trackRow)
+    {
+        return Db::name('shopro_miniprogram_track')
+            ->where('user_id', $trackRow['user_id'])
+            ->where('openid', $trackRow['openid'])
+            ->where('session_id', $trackRow['session_id'])
+            ->where('event_type', $trackRow['event_type'])
+            ->where('event_code', $trackRow['event_code'])
+            ->where('page_path', $trackRow['page_path'])
+            ->where('page_title', $trackRow['page_title'])
+            ->where('params', $trackRow['params'])
+            ->where('ip', $trackRow['ip'])
+            ->where('create_time', $trackRow['create_time'])
+            ->find() ? true : false;
+    }
+
 
 }
-?>
+?>
\ No newline at end of file

public/assets/js/backend/shopro/dashboard.js

@@ -54,7 +54,9 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form', 'moment'], function (
                         }, function (ret, res) { })
                     }
                     function initChartTotal(key) {
-                        var myChart = echarts.init(document.getElementById(`${key}Total`));
+                        var chartEl = document.getElementById(`${key}Total`);
+                        if (!chartEl) return;
+                        var myChart = echarts.init(chartEl);
                         window.onresize = () => {
                             myChart.resize()
                         }
@@ -370,14 +372,15 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form', 'moment'], function (
                             }
                             drawX(interval, kld);
                             console.log(chartOption, 'chartOption')
-                            var myChart2 = echarts.init(document.getElementById(`chartContent`));
-                            window.onresize = () => {
-                                myChart2.resize()
+                            var chartContentEl = document.getElementById(`chartContent`);
+                            if (chartContentEl) {
+                                var myChart2 = echarts.init(chartContentEl);
+                                window.onresize = () => {
+                                    myChart2.resize()
+                                }
+                                myChart2.setOption(chartOption);
                             }
 
-
-                            myChart2.setOption(chartOption);
-
                         } else {
                             chartOption.xAxis.data = [];
                             chartOption.series[0].data = [];
@@ -495,13 +498,15 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form', 'moment'], function (
                                 });
                             })
 
-                            var myChart3 = echarts.init(document.getElementById(`rankingContent`));
-                            window.onresize = () => {
-                                myChart3.resize()
+                            var rankingContentEl = document.getElementById(`rankingContent`);
+                            if (rankingContentEl) {
+                                var myChart3 = echarts.init(rankingContentEl);
+                                window.onresize = () => {
+                                    myChart3.resize()
+                                }
+                                myChart3.setOption(pieOption);
                             }
 
-                            myChart3.setOption(pieOption);
-
                             return false
                         }, function (ret, res) { })
                     }