Skip to content

L
loveisland
Commit 8da5c178 authored by 刘小敏's avatar 刘小敏
Browse files
Options

数据统计中心计划任务脚本，页面，接口

parent e7e8b184
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 197 additions and 23 deletions
addons/shopro/job/TrackProcessJob.php
View file @ 8da5c178
......@@ -7,6 +7,10 @@ use think\queue\Job;
use think\Db;
use think\Queue;
/**
* 埋点数据处理任务
* PV UV 等统计
*/
class TrackProcessJob
{
/**
......@@ -16,6 +20,7 @@ class TrackProcessJob
try {
$trackData = $data['data'] ?? [];
$clientIp = $data['ip'] ?? '';
$jobTimestamp = isset($data['timestamp']) ? (int)$data['timestamp'] : time();
if (empty($trackData)) {
$job->delete();
......@@ -23,7 +28,7 @@ class TrackProcessJob
}
// 批量插入埋点数据
$this->insertTracks($trackData, $clientIp);
$this->insertTracks($trackData, $clientIp, $jobTimestamp);
// 删除任务
$job->delete();
......@@ -43,22 +48,39 @@ class TrackProcessJob
/**
* 批量插入埋点数据
*/
protected function insertTracks($trackData, $clientIp) {
protected function insertTracks($trackData, $clientIp, $jobTimestamp) {
$insertData = [];
$now = time();
$seen = [];
foreach ($trackData as $item) {
$insertData[] = [
$trackRow = [
'user_id' => $item['user_id'] ?? 0,
'openid' => $item['openid'] ?? '',
'session_id' => $item['session_id'] ?? '',
'event_type' => $item['event_type'] ?? '',
'event_code' => $item['event_code'] ?? '',
'page_path' => $item['page_path'] ?? '',
'page_title' => $item['page_title'] ?? '',
'params' => !empty($item['params']) ? json_encode($item['params']) : '',
'ip' => $clientIp,
'create_time' => isset($item['create_time']) ? (int)($item['create_time'] / 1000) : $now
'create_time' => isset($item['create_time']) ? (int)($item['create_time'] / 1000) : $jobTimestamp
];
$dedupKey = $this->buildTrackDedupKey($trackRow);
if (isset($seen[$dedupKey])) {
continue;
}
$seen[$dedupKey] = true;
if ($this->trackExists($trackRow)) {
continue;
}
$insertData[] = $trackRow;
}
if (empty($insertData)) {
return;
}
// 分批插入（每批500条）
......@@ -69,10 +91,48 @@ class TrackProcessJob
}
/**
* 构建埋点判重键
*/
protected function buildTrackDedupKey($trackRow)
{
return md5(implode('|', [
(string)$trackRow['user_id'],
(string)$trackRow['openid'],
(string)$trackRow['session_id'],
(string)$trackRow['event_type'],
(string)$trackRow['event_code'],
(string)$trackRow['page_path'],
(string)$trackRow['page_title'],
(string)$trackRow['params'],
(string)$trackRow['ip'],
(string)$trackRow['create_time'],
]));
}
/**
* 判断埋点是否已存在
*/
protected function trackExists($trackRow)
{
return Db::name('shopro_miniprogram_track')
->where('user_id', $trackRow['user_id'])
->where('openid', $trackRow['openid'])
->where('session_id', $trackRow['session_id'])
->where('event_type', $trackRow['event_type'])
->where('event_code', $trackRow['event_code'])
->where('page_path', $trackRow['page_path'])
->where('page_title', $trackRow['page_title'])
->where('params', $trackRow['params'])
->where('ip', $trackRow['ip'])
->where('create_time', $trackRow['create_time'])
->find() ? true : false;
}
/**
* 保存失败数据
*/
protected function saveFailed($data, $error) {
Db::name('shopro_track_failed')->insert([
Db::name('shopro_miniprogram_track_failed')->insert([
'data' => json_encode($data),
'error' => $error,
'attempts' => 3,
......@@ -81,4 +141,4 @@ class TrackProcessJob
}
}
\ No newline at end of file
}
application/admin/view/shopro/common/script.html
View file @ 8da5c178
<link rel="stylesheet" href="//at.alicdn.com/t/c/font_2385137_b8qygb2jne.css?v={$site.version|htmlentities}">
<link rel="stylesheet" href="__CDN__/assets/addons/shopro/css/index.css?v={$site.version|htmlentities}">
<script src="__CDN__/assets/libs/jquery/dist/jquery.min.js?v={$site.version|htmlentities}"></script>
{if condition="$DARK_TYPE neq 'none'" }
<link rel="stylesheet" href="__CDN__/assets/addons/shopro/css/dark.css?v={$site.version|htmlentities}" data-render="darktheme" />
{/if}
......
application/command.php
View file @ 8da5c178
......@@ -10,6 +10,8 @@
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
use Dom\DtdNamedNodeMap;
return [
'app\admin\command\Crud',
'app\admin\command\Menu',
......@@ -21,4 +23,6 @@ return [
'app\command\StatMemberTag',
'app\command\StatsDaily',
'app\command\TrackFailedRetry',
'app\command\DataStatsCenter',
];
application/command/DataStatsCenter.php 0 → 100644
View file @ 8da5c178
This diff is collapsed.
application/command/TrackFailedRetry.php
View file @ 8da5c178
......@@ -36,6 +36,7 @@ class TrackFailedRetry extends Command
}
protected function execute(Input $input, Output $output) {
$this->output = $output;
$this->customLog('info', '========== [' . date('Y-m-d H:i:s') . '] 开始执行重试失败埋埋点数据任务 ==========');
......@@ -78,8 +79,24 @@ class TrackFailedRetry extends Command
throw new \Exception('数据格式错误');
}
$retryPayload = $this->filterRetryPayload($data);
if (empty($retryPayload['data'])) {
Db::name('shopro_miniprogram_track_failed')
->where('id', $item['id'])
->update([
'status' => 1,
'error' => '同一操作已入库，跳过重试',
'update_time' => time()
]);
$totalSuccess++;
$totalProcessed++;
continue;
}
// 重新推入队列
Queue::push('\addons\shopro\job\TrackProcessJob@fire', $data['data'], 'shopro-high');
Queue::push('\addons\shopro\job\TrackProcessJob@fire', $retryPayload, 'shopro-high');
// 更新状态为已处理
Db::name('shopro_miniprogram_track_failed')
......@@ -114,6 +131,94 @@ class TrackFailedRetry extends Command
$this->customLog('info', "重试完成！总计处理: {$totalProcessed} | 成功: {$totalSuccess} | 失败: {$totalFailed}");
}
/**
* 过滤已经入库的埋点，仅重试未入库数据
*/
protected function filterRetryPayload($payload)
{
$trackList = isset($payload['data']) && is_array($payload['data']) ? $payload['data'] : [];
$clientIp = $payload['ip'] ?? '';
$jobTimestamp = isset($payload['timestamp']) ? (int)$payload['timestamp'] : time();
$pendingList = [];
$seen = [];
foreach ($trackList as $trackItem) {
$trackRow = $this->buildTrackRow($trackItem, $clientIp, $jobTimestamp);
$dedupKey = $this->buildTrackDedupKey($trackRow);
if (isset($seen[$dedupKey])) {
continue;
}
$seen[$dedupKey] = true;
if ($this->trackExists($trackRow)) {
continue;
}
$pendingList[] = $trackItem;
}
$payload['data'] = $pendingList;
return $payload;
}
/**
* 构建埋点行数据
*/
protected function buildTrackRow($trackItem, $clientIp, $jobTimestamp)
{
return [
'user_id' => $trackItem['user_id'] ?? 0,
'openid' => $trackItem['openid'] ?? '',
'session_id' => $trackItem['session_id'] ?? '',
'event_type' => $trackItem['event_type'] ?? '',
'event_code' => $trackItem['event_code'] ?? '',
'page_path' => $trackItem['page_path'] ?? '',
'page_title' => $trackItem['page_title'] ?? '',
'params' => !empty($trackItem['params']) ? json_encode($trackItem['params']) : '',
'ip' => $clientIp,
'create_time' => isset($trackItem['create_time']) ? (int)($trackItem['create_time'] / 1000) : $jobTimestamp
];
}
/**
* 构建埋点判重键
*/
protected function buildTrackDedupKey($trackRow)
{
return md5(implode('|', [
(string)$trackRow['user_id'],
(string)$trackRow['openid'],
(string)$trackRow['session_id'],
(string)$trackRow['event_type'],
(string)$trackRow['event_code'],
(string)$trackRow['page_path'],
(string)$trackRow['page_title'],
(string)$trackRow['params'],
(string)$trackRow['ip'],
(string)$trackRow['create_time'],
]));
}
/**
* 判断埋点是否已存在
*/
protected function trackExists($trackRow)
{
return Db::name('shopro_miniprogram_track')
->where('user_id', $trackRow['user_id'])
->where('openid', $trackRow['openid'])
->where('session_id', $trackRow['session_id'])
->where('event_type', $trackRow['event_type'])
->where('event_code', $trackRow['event_code'])
->where('page_path', $trackRow['page_path'])
->where('page_title', $trackRow['page_title'])
->where('params', $trackRow['params'])
->where('ip', $trackRow['ip'])
->where('create_time', $trackRow['create_time'])
->find() ? true : false;
}
}
?>
\ No newline at end of file
?>
public/assets/js/backend/shopro/dashboard.js
View file @ 8da5c178
......@@ -54,7 +54,9 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form', 'moment'], function (
}, function (ret, res) { })
}
function initChartTotal(key) {
var myChart = echarts.init(document.getElementById(`${key}Total`));
var chartEl = document.getElementById(`${key}Total`);
if (!chartEl) return;
var myChart = echarts.init(chartEl);
window.onresize = () => {
myChart.resize()
}
......@@ -370,14 +372,15 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form', 'moment'], function (
}
drawX(interval, kld);
console.log(chartOption, 'chartOption')
var myChart2 = echarts.init(document.getElementById(`chartContent`));
window.onresize = () => {
myChart2.resize()
var chartContentEl = document.getElementById(`chartContent`);
if (chartContentEl) {
var myChart2 = echarts.init(chartContentEl);
window.onresize = () => {
myChart2.resize()
}
myChart2.setOption(chartOption);
}
myChart2.setOption(chartOption);
} else {
chartOption.xAxis.data = [];
chartOption.series[0].data = [];
......@@ -495,13 +498,15 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form', 'moment'], function (
});
})
var myChart3 = echarts.init(document.getElementById(`rankingContent`));
window.onresize = () => {
myChart3.resize()
var rankingContentEl = document.getElementById(`rankingContent`);
if (rankingContentEl) {
var myChart3 = echarts.init(rankingContentEl);
window.onresize = () => {
myChart3.resize()
}
myChart3.setOption(pieOption);
}
myChart3.setOption(pieOption);
return false
}, function (ret, res) { })
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment