Compare commits

...

24 Commits

Author SHA1 Message Date
nz 008fb370d5 install workflow 2026-05-20 16:38:08 +08:00
nz 444907aac4 update pw set 2026-05-20 16:18:57 +08:00
nz aa63d3db1c update playwright 2026-05-19 10:19:47 +08:00
nz 418c04980f update lock 2026-05-19 10:03:40 +08:00
nz 9107cf4651 upgrade playwright test deps to 1.60 2026-05-19 09:37:39 +08:00
nz 7a60ba4dab update test deps 2026-05-19 09:03:49 +08:00
nz 39c33273ef update backend build 2026-05-18 16:33:05 +08:00
nz 2a8365a8be update ignore 2026-05-18 16:14:25 +08:00
nz 13bacc6491 update 2026-05-18 16:10:05 +08:00
nz 05ac848293 update 2026-05-18 14:56:47 +08:00
nz d3f83023ea update env 2026-05-15 09:27:35 +08:00
nz 9d49b72abf update doc and mq.sh 2026-05-15 08:48:24 +08:00
admin 312abdc34b fix:修复rabbitmq.sh脚本执行后没有队列的问题 2026-05-13 14:07:12 +08:00
admin ddb8746954 添加安装部署脚本 2026-05-11 10:41:36 +08:00
nz eae665d66f add admin materialize 2026-05-07 21:25:38 +08:00
nz 349f8e11b0 add aggr 2026-05-07 20:51:38 +08:00
nz 1e7de46c26 add backfill 2026-05-07 16:00:10 +08:00
nz 597d8ae948 add daily paid view 2026-05-07 14:21:30 +08:00
nz 785726caac add timescaledb extension migration and orders_daily_by_created continuous aggregate
P22.1 of Stage 22 materialization layer infrastructure. Adds explicit
CREATE EXTENSION migration so clean environments can run `migrate` end-to-end
without ops manual setup, then creates the first continuous aggregate
orders_daily_by_created (day-bucketed, grouped by company/platform/store)
with WITH NO DATA. Five composite indexes cover the high-frequency query
patterns documented in docs/data_query.md. P22.2/P22.3 will add the
by-paid view, refresh policy and historical backfill on top.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-07 12:07:17 +08:00
nz efc5cabfbb update sku mapping 2026-04-20 13:50:20 +08:00
nz dd80286e23 update sku mapping 2026-04-20 13:26:48 +08:00
nz 93518fd031 add route group sync 2026-04-20 13:13:17 +08:00
nz 95ec0f16aa update 2026-04-20 11:06:55 +08:00
nz 7898beef5a update api key manage 2026-04-20 10:40:48 +08:00
82 changed files with 24691 additions and 860 deletions
+180
View File
@@ -0,0 +1,180 @@
name: build-and-push
on:
push:
tags:
- 'v*'
env:
REGISTRY: 192.168.30.181:3000
REPO_PATH: wpic-dev/datahub
jobs:
# ─────────────────────────────────────────────────────────────
# Pre-gate: tag must point to a commit reachable from master
# ─────────────────────────────────────────────────────────────
guard-master-only:
runs-on: podman
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Verify tag is on master
run: |
git fetch origin master:refs/remotes/origin/master
if ! git branch -r --contains "${{ gitea.sha }}" | grep -qE '(^|\s)origin/master(\s|$)'; then
echo "::error::Tag ${{ gitea.ref_name }} (commit ${{ gitea.sha }}) is NOT on master branch. Refusing to build."
exit 1
fi
echo "OK: tag ${{ gitea.ref_name }} is on master"
# ─────────────────────────────────────────────────────────────
# Test gates
# ─────────────────────────────────────────────────────────────
test-backend:
runs-on: podman
needs: [guard-master-only]
container:
image: docker.io/hyperf/hyperf:8.3-alpine-v3.19-swoole
steps:
- uses: actions/checkout@v4
- name: composer install (incl. dev for phpstan)
run: cd backend && composer install --no-interaction --no-progress
- name: PHP syntax check (parallel)
run: |
cd backend && find app config bin migrations -name '*.php' -print0 \
| xargs -0 -n1 -P4 php -l
- name: phpstan static analysis
run: cd backend && composer analyse
test-frontend-build:
runs-on: podman
needs: [guard-master-only]
container:
image: docker.io/library/node:22-alpine
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
steps:
- uses: actions/checkout@v4
- name: npm ci
run: cd frontend && npm ci --no-audit --no-fund
- name: lint (pre-existing baseline tolerated)
run: cd frontend && npm run lint
continue-on-error: true
- name: type-check (vue-tsc)
run: cd frontend && npm run type-check
- name: unit tests (pre-existing baseline tolerated)
run: cd frontend && npm run test:unit -- --run
continue-on-error: true
- name: build (vite)
run: cd frontend && npm run build
- name: upload artifacts
uses: actions/upload-artifact@v3
with:
name: frontend-build
path: |
frontend/dist
frontend/node_modules
test-frontend-e2e:
runs-on: podman
needs: [test-frontend-build]
container:
image: mcr.microsoft.com/playwright:v1.60.0-noble
steps:
- uses: actions/checkout@v4
- name: download artifacts
uses: actions/download-artifact@v3
with:
name: frontend-build
path: frontend
- name: playwright smoke (chromium only)
run: cd frontend && CI=true npx playwright test --project=chromium
# ─────────────────────────────────────────────────────────────
# Build & push (4 images, parallel where possible)
# ─────────────────────────────────────────────────────────────
build-backend:
runs-on: podman
needs: [test-backend]
steps:
- uses: actions/checkout@v4
- name: Login to Gitea Registry
run: |
echo "${{ secrets.DATAHUB_CI_CD }}" | \
podman login -u "${{ gitea.actor }}" --password-stdin ${{ env.REGISTRY }}
- name: Build
run: |
podman build --pull --layers \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/backend:${{ gitea.ref_name }} \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/backend:stable \
-f backend/Dockerfile \
backend/
- name: Push
run: |
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/backend:${{ gitea.ref_name }}
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/backend:stable
build-frontend:
runs-on: podman
needs: [test-frontend-e2e]
steps:
- uses: actions/checkout@v4
- name: Login to Gitea Registry
run: |
echo "${{ secrets.DATAHUB_CI_CD }}" | \
podman login -u "${{ gitea.actor }}" --password-stdin ${{ env.REGISTRY }}
- name: Build
run: |
podman build --pull --layers \
--ulimit nofile=65536:65536 \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/frontend:${{ gitea.ref_name }} \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/frontend:stable \
-f frontend/Dockerfile \
frontend/
- name: Push
run: |
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/frontend:${{ gitea.ref_name }}
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/frontend:stable
build-timescaledb2:
runs-on: podman
needs: [guard-master-only]
steps:
- uses: actions/checkout@v4
- name: Login to Gitea Registry
run: |
echo "${{ secrets.DATAHUB_CI_CD }}" | \
podman login -u "${{ gitea.actor }}" --password-stdin ${{ env.REGISTRY }}
- name: Build
run: |
podman build --pull --layers \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/timescaledb2:${{ gitea.ref_name }} \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/timescaledb2:stable \
-f docs/tmp/deploy-ref/ci-cd/03-timescaledb-image/Containerfile \
docs/tmp/deploy-ref/ci-cd/03-timescaledb-image/
- name: Push
run: |
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/timescaledb2:${{ gitea.ref_name }}
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/timescaledb2:stable
build-rabbitmq3:
runs-on: podman
needs: [guard-master-only]
steps:
- uses: actions/checkout@v4
- name: Login to Gitea Registry
run: |
echo "${{ secrets.DATAHUB_CI_CD }}" | \
podman login -u "${{ gitea.actor }}" --password-stdin ${{ env.REGISTRY }}
- name: Build
run: |
podman build --pull --layers \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/rabbitmq3:${{ gitea.ref_name }} \
-t ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/rabbitmq3:stable \
-f docs/tmp/deploy-ref/ci-cd/04-rabbitmq-image/Containerfile \
docs/tmp/deploy-ref/ci-cd/04-rabbitmq-image/
- name: Push
run: |
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/rabbitmq3:${{ gitea.ref_name }}
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/rabbitmq3:stable
+65
View File
@@ -0,0 +1,65 @@
name: rollback
on:
workflow_dispatch:
inputs:
target_tag:
description: 'Target release tag to rollback to (e.g. v1.2.3)'
required: true
component:
description: 'Component (backend / frontend / timescaledb2 / rabbitmq3 / all)'
required: true
default: 'all'
env:
REGISTRY: 192.168.30.181:3000
REPO_PATH: wpic-dev/datahub
jobs:
retag:
runs-on: podman
steps:
- name: Validate target_tag format
run: |
if [[ ! "${{ inputs.target_tag }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]]; then
echo "::error::target_tag must match semver pattern v<MAJOR>.<MINOR>.<PATCH>[-suffix]"
exit 1
fi
- name: Validate component
run: |
case "${{ inputs.component }}" in
backend|frontend|timescaledb2|rabbitmq3|all) ;;
*)
echo "::error::component must be one of: backend / frontend / timescaledb2 / rabbitmq3 / all"
exit 1
;;
esac
- name: Login to Gitea Registry
run: |
echo "${{ secrets.DATAHUB_CI_CD }}" | \
podman login -u "${{ gitea.actor }}" --password-stdin ${{ env.REGISTRY }}
- name: Re-tag stable to target release
run: |
set -euo pipefail
components_to_rollback=""
if [[ "${{ inputs.component }}" == "all" ]]; then
components_to_rollback="backend frontend timescaledb2 rabbitmq3"
else
components_to_rollback="${{ inputs.component }}"
fi
for img in $components_to_rollback; do
echo "=== Rolling back $img to ${{ inputs.target_tag }} ==="
podman pull ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/$img:${{ inputs.target_tag }}
podman tag ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/$img:${{ inputs.target_tag }} \
${{ env.REGISTRY }}/${{ env.REPO_PATH }}/$img:stable
podman push ${{ env.REGISTRY }}/${{ env.REPO_PATH }}/$img:stable
echo "✓ $img:stable now points to ${{ inputs.target_tag }}"
done
echo
echo "Rollback complete. wpic-virt podman-auto-update.timer will pull the new"
echo ":stable digest within ~5 minutes and restart affected containers."
-5
View File
@@ -14,7 +14,6 @@ backend/phpstan.neon
backend/phpunit.xml
backend/.php_cs.cache
backend/.php-cs-fixer.cache
backend/composer.lock
# ==================== 前端 (Vue3/Vite) ====================
frontend/node_modules/
@@ -25,9 +24,6 @@ frontend/coverage/
frontend/*.local
frontend/.eslintcache
frontend/*.tsbuildinfo
frontend/package-lock.json
frontend/pnpm-lock.yaml
frontend/yarn.lock
# 前端日志文件
frontend/logs/
@@ -152,7 +148,6 @@ $RECYCLE.BIN/
# ==================== Docker ====================
docker-compose.override.yml
.dockerignore
# ==================== 版本控制 ====================
*.patch
+7
View File
@@ -0,0 +1,7 @@
**
!app/
!bin/
!config/
!migrations/
!composer.*
!entrypoint.sh
+2 -2
View File
@@ -1,4 +1,4 @@
APP_NAME=skeleton
APP_NAME=dataghub
APP_ENV=dev
DB_DRIVER=mysql
@@ -17,7 +17,7 @@ AMQP_ADMIN_USER="user"
AMQP_ADMIN_PASSWORD="pass"
AMQP_USER="user_tmall"
AMQP_PASSWORD="change_me_tmall"
AMQP_VHOST="dataflow"
AMQP_VHOST="datahub"
# 消息最大重试次数(默认3次),超过后进入错误队列
AMQP_MAX_RETRIES=3
# 调试模式:消费者处理每条消息的延迟秒数(默认0,设置为2可方便在mq:status中观察)
+31 -40
View File
@@ -1,54 +1,45 @@
# Default Dockerfile
#
# @link https://www.hyperf.io
# @document https://hyperf.wiki
# @contact group@hyperf.io
# @license https://github.com/hyperf/hyperf/blob/master/LICENSE
FROM docker.io/hyperf/hyperf:8.3-alpine-v3.19-swoole
FROM hyperf/hyperf:8.3-alpine-v3.19-swoole
LABEL maintainer="Hyperf Developers <group@hyperf.io>" version="1.0" license="MIT" app.name="Hyperf"
LABEL org.opencontainers.image.title="datahub-backend" \
org.opencontainers.image.vendor="WPIC" \
org.opencontainers.image.licenses="MIT" \
org.opencontainers.image.source="https://192.168.30.181:3000/wpic-dev/datahub"
##
# ---------- env settings ----------
##
# --build-arg timezone=Asia/Shanghai
ARG timezone
ENV TIMEZONE=${timezone:-"Asia/Shanghai"} \
ARG TIMEZONE=Asia/Shanghai
ENV TIMEZONE=${TIMEZONE} \
APP_ENV=prod \
SCAN_CACHEABLE=(true)
# update
RUN set -ex \
# show php version and extensions
&& php -v \
&& php -m \
&& php --ri swoole \
# ---------- some config ----------
&& cd /etc/php* \
# - config PHP
&& { \
RUN apk add --no-cache php83-intl php83-pspell
RUN set -eux; \
php -v; php -m; php --ri swoole; \
cd /etc/php*; \
{ \
echo "upload_max_filesize=128M"; \
echo "post_max_size=128M"; \
echo "memory_limit=1G"; \
echo "memory_limit=2G"; \
echo "date.timezone=${TIMEZONE}"; \
} | tee conf.d/99_overrides.ini \
# - config timezone
&& ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime \
&& echo "${TIMEZONE}" > /etc/timezone \
# ---------- clear works ----------
&& rm -rf /var/cache/apk/* /tmp/* /usr/share/man \
&& echo -e "\033[42;37m Build Completed :).\033[0m\n"
} | tee conf.d/99_overrides.ini; \
ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime; \
echo "${TIMEZONE}" > /etc/timezone; \
rm -rf /var/cache/apk/* /tmp/* /usr/share/man
WORKDIR /opt/www
WORKDIR /var/www
# Composer Cache
# COPY ./composer.* /opt/www/
# RUN composer install --no-dev --no-scripts
COPY composer.json composer.lock ./
RUN composer install --no-dev --no-scripts --no-autoloader --prefer-dist --no-interaction
COPY . /opt/www
RUN composer install --no-dev -o && php bin/hyperf.php
COPY . .
RUN composer dump-autoload --optimize --classmap-authoritative --no-dev
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh
EXPOSE 9501
ENTRYPOINT ["php", "/opt/www/bin/hyperf.php", "start"]
HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
CMD wget -qO- http://127.0.0.1:9501/health || exit 1
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["php", "/var/www/bin/hyperf.php", "start"]
@@ -0,0 +1,46 @@
<?php
declare(strict_types=1);
namespace App\Command;
use Hyperf\Command\Annotation\Command;
use Hyperf\Command\Command as HyperfCommand;
use Hyperf\DbConnection\Db;
#[Command]
class OrderAggregatesBackfillCommand extends HyperfCommand
{
public function __construct()
{
parent::__construct('orders:backfill-aggregates');
}
public function configure(): void
{
parent::configure();
$this->setDescription('一次性回填 orders_daily_by_created(连续聚合)和 orders_daily_by_paid(物化视图)的全部历史数据');
}
public function handle(): void
{
// 1. orders_daily_by_created:调用 TimescaleDB 内置 refresh
$this->line('Refreshing orders_daily_by_created (NULL → now() - 1 hour) ...');
Db::statement("CALL refresh_continuous_aggregate('orders_daily_by_created', NULL, now() - INTERVAL '1 hour')");
// 2. orders_daily_by_paidPG 物化视图。首次必须用非 CONCURRENTLY 模式填充,
// 后续重算才能走 CONCURRENTLYPG 硬约束:CONCURRENTLY cannot be used when not populated)。
$rows = Db::select("SELECT ispopulated FROM pg_matviews WHERE matviewname = 'orders_daily_by_paid'");
$populated = ! empty($rows) && $rows[0]->ispopulated;
if ($populated) {
$this->line('Refreshing orders_daily_by_paid (CONCURRENTLY) ...');
Db::statement('REFRESH MATERIALIZED VIEW CONCURRENTLY orders_daily_by_paid');
} else {
$this->line('Initial population of orders_daily_by_paid (non-concurrent) ...');
Db::statement('REFRESH MATERIALIZED VIEW orders_daily_by_paid');
}
$this->info('Done.');
}
}
@@ -0,0 +1,35 @@
<?php
declare(strict_types=1);
namespace App\Command;
use App\Service\OrderAggregatesRefreshJob;
use Hyperf\Command\Annotation\Command;
use Hyperf\Command\Command as HyperfCommand;
/**
* 手动触发聚合刷新队列消费
*
* 与 Crontab 共用 OrderAggregatesRefreshJob,便于运维即时补刷或调试。
*/
#[Command]
class OrderAggregatesRefreshCommand extends HyperfCommand
{
public function __construct(private OrderAggregatesRefreshJob $job)
{
parent::__construct('orders:refresh-aggregates');
}
public function configure(): void
{
parent::configure();
$this->setDescription('消费 aggregate_refresh_queue,对滞后日期调用 refresh_continuous_aggregate');
}
public function handle(): void
{
$result = $this->job->run();
$this->info(sprintf('Processed: %d, Failed: %d', $result['processed'], $result['failed']));
}
}
@@ -37,6 +37,7 @@ class RouteGroupSeedCommand extends HyperfCommand
['name' => 'user-permission', 'label' => '用户与权限', 'sort_order' => 8, 'patterns' => ['/api/v1/users', '/api/v1/roles', '/api/v1/route-groups', '/api/v1/routes']],
['name' => 'platform-management', 'label' => '平台管理', 'sort_order' => 9, 'patterns' => ['/api/v1/platforms']],
['name' => 'system', 'label' => '系统功能', 'sort_order' => 10, 'patterns' => ['/api/v1/me/', '/api/v1/dashboard', '/api/v1/mq', '/api/v1/failed-messages', '/api/v1/auth/']],
['name' => 'materialization-admin', 'label' => '物化任务管理', 'sort_order' => 11, 'patterns' => ['/api/v1/admin/materialization/']],
];
$group_count = 0;
@@ -21,18 +21,19 @@ class AdminApiKeyController extends AbstractController
/**
* 管理员列出所有 API Keys
*
* 支持按 user_id、enabled 筛选,关联用户信息
* 支持按 username、email、enabled 筛选,关联用户信息(含 email
*/
#[OA\Get(
path: '/admin/api-keys',
summary: '管理员列出所有 API Keys',
description: '分页列出所有用户的 API Keys,支持按 user_id、enabled 筛选,关联用户基本信息',
description: '分页列出所有用户的 API Keys,支持按 username、email、enabled 筛选,关联用户基本信息',
security: [['bearerAuth' => []]],
tags: ['Admin API Keys'],
parameters: [
new OA\Parameter(name: 'page', in: 'query', required: false, description: '页码,默认 1', schema: new OA\Schema(type: 'integer', default: 1)),
new OA\Parameter(name: 'per_page', in: 'query', required: false, description: '每页条数,默认 15,最大 100', schema: new OA\Schema(type: 'integer', default: 15)),
new OA\Parameter(name: 'user_id', in: 'query', required: false, description: '按用户 ID 筛选', schema: new OA\Schema(type: 'integer')),
new OA\Parameter(name: 'username', in: 'query', required: false, description: '按用户名模糊搜索', schema: new OA\Schema(type: 'string')),
new OA\Parameter(name: 'email', in: 'query', required: false, description: '按邮箱模糊搜索', schema: new OA\Schema(type: 'string')),
new OA\Parameter(name: 'enabled', in: 'query', required: false, description: '按启用状态筛选(0/1', schema: new OA\Schema(type: 'integer', enum: [0, 1])),
],
responses: [
@@ -55,6 +56,7 @@ class AdminApiKeyController extends AbstractController
new OA\Property(property: 'user', properties: [
new OA\Property(property: 'id', type: 'integer'),
new OA\Property(property: 'username', type: 'string'),
new OA\Property(property: 'email', type: 'string'),
new OA\Property(property: 'api_key_enabled', type: 'boolean'),
], type: 'object'),
])),
@@ -71,17 +73,29 @@ class AdminApiKeyController extends AbstractController
#[RequestMapping(path: "", methods: "GET")]
#[Middleware(AuthMiddleware::class)]
#[Middleware(PermissionMiddleware::class)]
public function index(): array
public function index(): ResponseInterface|array
{
if ($forbidden = $this->requireAdmin()) return $forbidden;
$page = (int) $this->request->input('page', 1);
$per_page = min((int) $this->request->input('per_page', 15), 100);
$query = ApiKey::query()->with('user:id,username,api_key_enabled');
$query = ApiKey::query()->with('user:id,username,email,api_key_enabled');
// 按用户 ID 筛选
$user_id = $this->request->input('user_id');
if ($user_id !== null && $user_id !== '') {
$query->where('user_id', (int) $user_id);
// 按用户名模糊搜索
$username = $this->request->input('username');
if ($username !== null && $username !== '') {
$query->whereHas('user', function ($q) use ($username) {
$q->where('username', 'like', '%' . $username . '%');
});
}
// 按邮箱模糊搜索
$email = $this->request->input('email');
if ($email !== null && $email !== '') {
$query->whereHas('user', function ($q) use ($email) {
$q->where('email', 'like', '%' . $email . '%');
});
}
// 按启用状态筛选
@@ -155,6 +169,8 @@ class AdminApiKeyController extends AbstractController
#[Middleware(PermissionMiddleware::class)]
public function toggle(int $id): ResponseInterface|array
{
if ($forbidden = $this->requireAdmin()) return $forbidden;
$api_key = ApiKey::query()->find($id);
if (!$api_key) {
@@ -213,6 +229,8 @@ class AdminApiKeyController extends AbstractController
#[Middleware(PermissionMiddleware::class)]
public function destroy(int $id): ResponseInterface|array
{
if ($forbidden = $this->requireAdmin()) return $forbidden;
$api_key = ApiKey::query()->find($id);
if (!$api_key) {
@@ -229,4 +247,16 @@ class AdminApiKeyController extends AbstractController
'message' => '删除成功',
];
}
private function requireAdmin(): ?ResponseInterface
{
$user = $this->getAuthUser();
if (!$user || !$user->isAdministrator()) {
return $this->response->json([
'code' => 403,
'message' => '仅管理员可访问',
])->withStatus(403);
}
return null;
}
}
@@ -0,0 +1,346 @@
<?php
declare(strict_types=1);
namespace App\Controller\Api\V1;
use App\Controller\AbstractController;
use App\Middleware\AuthMiddleware;
use App\Middleware\PermissionMiddleware;
use App\Model\AggregateRefreshQueue;
use Hyperf\DbConnection\Db;
use Hyperf\HttpServer\Annotation\Controller;
use Hyperf\HttpServer\Annotation\Middleware;
use Hyperf\HttpServer\Annotation\RequestMapping;
use OpenApi\Attributes as OA;
use Psr\Http\Message\ResponseInterface;
#[OA\Tag(name: 'Admin Materialization', description: '管理员物化层监控')]
#[Controller(prefix: "/api/v1/admin/materialization")]
class AdminMaterializationController extends AbstractController
{
private const VIEW_BY_CREATED = 'orders_daily_by_created';
private const VIEW_BY_PAID = 'orders_daily_by_paid';
private const ALLOWED_VIEWS = [self::VIEW_BY_CREATED, self::VIEW_BY_PAID];
/**
* 列出 aggregate_refresh_queue 待刷新条目
*/
#[OA\Get(
path: '/admin/materialization/queue',
summary: '列出聚合刷新队列',
description: '分页列出 aggregate_refresh_queue 表中的待刷新日期,支持按 view/from/to 过滤',
security: [['bearerAuth' => []]],
tags: ['Admin Materialization'],
parameters: [
new OA\Parameter(name: 'page', in: 'query', required: false, schema: new OA\Schema(type: 'integer', default: 1)),
new OA\Parameter(name: 'per_page', in: 'query', required: false, schema: new OA\Schema(type: 'integer', default: 20)),
new OA\Parameter(name: 'view', in: 'query', required: false, description: '聚合视图名(白名单)', schema: new OA\Schema(type: 'string', enum: self::ALLOWED_VIEWS)),
new OA\Parameter(name: 'from', in: 'query', required: false, description: 'refresh_date >= fromYYYY-MM-DD', schema: new OA\Schema(type: 'string', format: 'date')),
new OA\Parameter(name: 'to', in: 'query', required: false, description: 'refresh_date <= toYYYY-MM-DD', schema: new OA\Schema(type: 'string', format: 'date')),
],
responses: [
new OA\Response(
response: 200,
description: '获取成功',
content: new OA\JsonContent(properties: [
new OA\Property(property: 'code', type: 'integer', example: 0),
new OA\Property(property: 'message', type: 'string', example: '获取成功'),
new OA\Property(property: 'data', properties: [
new OA\Property(property: 'items', type: 'array', items: new OA\Items(properties: [
new OA\Property(property: 'refresh_date', type: 'string', format: 'date'),
new OA\Property(property: 'aggregate_view', type: 'string'),
new OA\Property(property: 'created_at', type: 'string', format: 'date-time'),
])),
new OA\Property(property: 'total', type: 'integer'),
new OA\Property(property: 'page', type: 'integer'),
new OA\Property(property: 'per_page', type: 'integer'),
], type: 'object'),
])
),
new OA\Response(response: 401, description: '未认证', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
new OA\Response(response: 403, description: '无权限', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
]
)]
#[RequestMapping(path: "queue", methods: "GET")]
#[Middleware(AuthMiddleware::class)]
#[Middleware(PermissionMiddleware::class)]
public function queue(): ResponseInterface|array
{
if ($forbidden = $this->requireAdmin()) return $forbidden;
$page = max((int) $this->request->input('page', 1), 1);
$per_page = min(max((int) $this->request->input('per_page', 20), 1), 100);
$view = $this->request->input('view');
$from = $this->request->input('from');
$to = $this->request->input('to');
$query = AggregateRefreshQueue::query();
if ($view !== null && $view !== '') {
$query->where('aggregate_view', $view);
}
if ($from !== null && $from !== '') {
$query->where('refresh_date', '>=', $from);
}
if ($to !== null && $to !== '') {
$query->where('refresh_date', '<=', $to);
}
$total = $query->count();
$items = $query
->orderBy('refresh_date')
->orderBy('aggregate_view')
->offset(($page - 1) * $per_page)
->limit($per_page)
->get();
return [
'code' => 0,
'message' => '获取成功',
'data' => [
'items' => $items,
'total' => $total,
'page' => $page,
'per_page' => $per_page,
],
];
}
/**
* 手动触发指定时间窗口的物化刷新
*/
#[OA\Post(
path: '/admin/materialization/refresh',
summary: '手动刷新物化对象',
description: 'by_created 走 refresh_continuous_aggregate(view, from, to)by_paid 是 PG 物化视图,from/to 无效,整体走 REFRESH MATERIALIZED VIEW [CONCURRENTLY]',
security: [['bearerAuth' => []]],
tags: ['Admin Materialization'],
requestBody: new OA\RequestBody(
required: true,
content: new OA\JsonContent(
required: ['view'],
properties: [
new OA\Property(property: 'view', type: 'string', enum: self::ALLOWED_VIEWS),
new OA\Property(property: 'from', type: 'string', description: '起始时间戳(仅 by_created 生效,timestamptz 字面量)'),
new OA\Property(property: 'to', type: 'string', description: '结束时间戳(仅 by_created 生效,timestamptz 字面量)'),
]
)
),
responses: [
new OA\Response(
response: 200,
description: '刷新成功',
content: new OA\JsonContent(properties: [
new OA\Property(property: 'code', type: 'integer', example: 0),
new OA\Property(property: 'message', type: 'string', example: '刷新成功'),
new OA\Property(property: 'data', properties: [
new OA\Property(property: 'view', type: 'string'),
new OA\Property(property: 'from', type: 'string', nullable: true),
new OA\Property(property: 'to', type: 'string', nullable: true),
new OA\Property(property: 'mode', type: 'string', enum: ['incremental', 'full_refresh', 'full_refresh_concurrent']),
], type: 'object'),
])
),
new OA\Response(response: 400, description: '参数错误或刷新失败', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
new OA\Response(response: 401, description: '未认证', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
new OA\Response(response: 403, description: '无权限', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
]
)]
#[RequestMapping(path: "refresh", methods: "POST")]
#[Middleware(AuthMiddleware::class)]
#[Middleware(PermissionMiddleware::class)]
public function refresh(): ResponseInterface|array
{
if ($forbidden = $this->requireAdmin()) return $forbidden;
$view = (string) $this->request->input('view', '');
$from = $this->request->input('from');
$to = $this->request->input('to');
if (!in_array($view, self::ALLOWED_VIEWS, true)) {
return $this->response->json([
'code' => 400,
'message' => 'invalid view: must be one of ' . implode(', ', self::ALLOWED_VIEWS),
'data' => null,
])->withStatus(400);
}
try {
if ($view === self::VIEW_BY_CREATED) {
Db::statement(
'CALL refresh_continuous_aggregate(?, ?::timestamptz, ?::timestamptz)',
[$view, $from, $to]
);
return [
'code' => 0,
'message' => '刷新成功',
'data' => [
'view' => $view,
'from' => $from,
'to' => $to,
'mode' => 'incremental',
],
];
}
// by_paidPG 物化视图,from/to 无效
$rows = Db::select("SELECT ispopulated FROM pg_matviews WHERE matviewname = ?", [self::VIEW_BY_PAID]);
$populated = !empty($rows) && $rows[0]->ispopulated;
$mode = $populated ? 'full_refresh_concurrent' : 'full_refresh';
$sql = $populated
? 'REFRESH MATERIALIZED VIEW CONCURRENTLY ' . self::VIEW_BY_PAID
: 'REFRESH MATERIALIZED VIEW ' . self::VIEW_BY_PAID;
Db::statement($sql);
return [
'code' => 0,
'message' => '刷新成功',
'data' => [
'view' => $view,
'from' => null,
'to' => null,
'mode' => $mode,
],
];
} catch (\Throwable $e) {
return $this->response->json([
'code' => 400,
'message' => 'refresh failed: ' . $e->getMessage(),
'data' => null,
])->withStatus(400);
}
}
/**
* 列出连续聚合视图及其滞后秒数
*/
#[OA\Get(
path: '/admin/materialization/aggregates',
summary: '查询连续聚合滞后情况',
description: '查询 timescaledb_information.continuous_aggregates,附加由 cagg_watermark 计算的 lag_seconds 字段',
security: [['bearerAuth' => []]],
tags: ['Admin Materialization'],
responses: [
new OA\Response(
response: 200,
description: '获取成功',
content: new OA\JsonContent(properties: [
new OA\Property(property: 'code', type: 'integer', example: 0),
new OA\Property(property: 'message', type: 'string', example: '获取成功'),
new OA\Property(property: 'data', properties: [
new OA\Property(property: 'items', type: 'array', items: new OA\Items(properties: [
new OA\Property(property: 'view_name', type: 'string'),
new OA\Property(property: 'view_schema', type: 'string'),
new OA\Property(property: 'materialization_hypertable_name', type: 'string'),
new OA\Property(property: 'lag_seconds', type: 'number', format: 'double', nullable: true),
])),
], type: 'object'),
])
),
new OA\Response(response: 401, description: '未认证', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
new OA\Response(response: 403, description: '无权限', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
]
)]
#[RequestMapping(path: "aggregates", methods: "GET")]
#[Middleware(AuthMiddleware::class)]
#[Middleware(PermissionMiddleware::class)]
public function aggregates(): ResponseInterface|array
{
if ($forbidden = $this->requireAdmin()) return $forbidden;
$rows = Db::select(
"SELECT ca.view_name,
ca.view_schema,
ca.materialization_hypertable_name,
EXTRACT(EPOCH FROM (now() - to_timestamp(_timescaledb_functions.cagg_watermark(cagg.mat_hypertable_id)::float8 / 1000000))) AS lag_seconds
FROM timescaledb_information.continuous_aggregates ca
JOIN _timescaledb_catalog.continuous_agg cagg
ON cagg.user_view_name = ca.view_name
AND cagg.user_view_schema = ca.view_schema
ORDER BY ca.view_name"
);
return [
'code' => 0,
'message' => '获取成功',
'data' => ['items' => $rows],
];
}
/**
* 列出 TimescaleDB Crontab 任务及其最近执行状态
*/
#[OA\Get(
path: '/admin/materialization/jobs',
summary: '查询 Crontab 任务状态',
description: '查询 timescaledb_information.jobs LEFT JOIN job_stats,仅 policy_refresh_continuous_aggregate 类型',
security: [['bearerAuth' => []]],
tags: ['Admin Materialization'],
responses: [
new OA\Response(
response: 200,
description: '获取成功',
content: new OA\JsonContent(properties: [
new OA\Property(property: 'code', type: 'integer', example: 0),
new OA\Property(property: 'message', type: 'string', example: '获取成功'),
new OA\Property(property: 'data', properties: [
new OA\Property(property: 'items', type: 'array', items: new OA\Items(properties: [
new OA\Property(property: 'job_id', type: 'integer'),
new OA\Property(property: 'application_name', type: 'string'),
new OA\Property(property: 'proc_name', type: 'string'),
new OA\Property(property: 'hypertable_name', type: 'string', nullable: true),
new OA\Property(property: 'schedule_interval', type: 'string'),
new OA\Property(property: 'next_start', type: 'string', format: 'date-time', nullable: true),
new OA\Property(property: 'last_successful_finish', type: 'string', format: 'date-time', nullable: true),
new OA\Property(property: 'last_run_status', type: 'string', nullable: true),
])),
], type: 'object'),
])
),
new OA\Response(response: 401, description: '未认证', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
new OA\Response(response: 403, description: '无权限', content: new OA\JsonContent(ref: '#/components/schemas/ErrorResponse')),
]
)]
#[RequestMapping(path: "jobs", methods: "GET")]
#[Middleware(AuthMiddleware::class)]
#[Middleware(PermissionMiddleware::class)]
public function jobs(): ResponseInterface|array
{
if ($forbidden = $this->requireAdmin()) return $forbidden;
$rows = Db::select(
"SELECT j.job_id,
j.application_name,
j.proc_name,
j.hypertable_name,
j.schedule_interval,
j.next_start,
s.last_successful_finish,
s.last_run_status
FROM timescaledb_information.jobs j
LEFT JOIN timescaledb_information.job_stats s ON s.job_id = j.job_id
WHERE j.proc_name = 'policy_refresh_continuous_aggregate'
ORDER BY j.job_id"
);
return [
'code' => 0,
'message' => '获取成功',
'data' => ['items' => $rows],
];
}
private function requireAdmin(): ?ResponseInterface
{
$user = $this->getAuthUser();
if (!$user || !$user->isAdministrator()) {
return $this->response->json([
'code' => 403,
'message' => '仅管理员可访问',
])->withStatus(403);
}
return null;
}
}
@@ -0,0 +1,31 @@
<?php
declare(strict_types=1);
namespace App\Model;
/**
* @property string $refresh_date 格式 Y-m-d
* @property string $aggregate_view 视图名(如 orders_daily_by_created
* @property \Carbon\Carbon $created_at 入队时间
*/
class AggregateRefreshQueue extends Model
{
protected ?string $table = 'aggregate_refresh_queue';
public bool $timestamps = false;
public bool $incrementing = false;
protected string $primaryKey = 'refresh_date';
protected array $fillable = [
'refresh_date',
'aggregate_view',
'created_at',
];
protected array $casts = [
'created_at' => 'datetime',
];
}
+51 -1
View File
@@ -5,8 +5,10 @@ declare(strict_types=1);
namespace App\Platform;
use App\Entity\Parse\EntityParseFactory;
use App\Model\AggregateRefreshQueue;
use App\Platform\Traits\FailedMessageTrait;
use App\Utils\Log;
use Carbon\Carbon;
use Hyperf\Amqp\Annotation\Consumer;
use Hyperf\Amqp\Builder\QueueBuilder;
use Hyperf\Amqp\Message\ConsumerMessage;
@@ -152,9 +154,10 @@ class OrderConsumer extends ConsumerMessage
// 鉴于定义子项为了保留足够的灵活性,因此每次订单更新,我们都需要完整更新 OrderItem
$this->processOrderItems($items);
// 5. 识别 ≥ 3 天前的 created_date 入队,补刷自动策略未覆盖的窗口
$this->enqueueAffectedDates($orders_data);
Db::commit();
// @TODO 触发事件通知,更新自动聚合任务
// 在数据库事务中尝试对 $entityMapResult 中的元素进行持久化,如果没有问题, 则返回 ACK,否则这是 NACK 且 回滚事务。
return Result::ACK;
@@ -379,4 +382,51 @@ class OrderConsumer extends ConsumerMessage
}
}
/**
* 识别 payload 中 ≥ 3 天前的 created_date,入队 orders_daily_by_created 兜底刷新。
*
* 自动刷新策略仅覆盖最近 3 天窗口;3 天前的订单变更(补录、追溯调整)需由
* aggregate_refresh_queue + Crontab 任务补刷。仅服务 by_created 视图,
* by_paid 由全量 REFRESH 覆盖,不入此队列。
*
* @param array $payloads 来自 entityMap()->all() 的订单数组,每条含 created_date 字段
*/
protected function enqueueAffectedDates(array $payloads): void
{
if (empty($payloads)) {
return;
}
$threshold = Carbon::now()->subDays(3)->toDateString();
$unique_dates = [];
foreach ($payloads as $payload) {
$created = $payload['created_date'] ?? null;
if ($created === null) {
continue;
}
// entityMap 输出 'Y-m-d H:i:sP';用 Carbon::parse 兼容多种格式
$date = Carbon::parse($created)->toDateString();
// 严格小于阈值才入队(≥ 阈值的部分由自动刷新策略覆盖)
if ($date < $threshold) {
$unique_dates[$date] = true;
}
}
if (empty($unique_dates)) {
return;
}
$now = Carbon::now();
foreach (array_keys($unique_dates) as $date) {
AggregateRefreshQueue::query()->insertOrIgnore([
'refresh_date' => $date,
'aggregate_view' => 'orders_daily_by_created',
'created_at' => $now,
]);
}
}
}
@@ -0,0 +1,21 @@
<?php
declare(strict_types=1);
namespace App\Service;
/**
* 聚合视图刷新接口
*
* 隔离 PG procedure 调用,便于在单测中注入 mock 实现。
*/
interface AggregateRefresherInterface
{
/**
* 对指定聚合视图的指定日期刷新
*
* @param string $view 聚合视图名(如 'orders_daily_by_created'
* @param string $refresh_date Y-m-d 格式日期
*/
public function refresh(string $view, string $refresh_date): void;
}
@@ -0,0 +1,31 @@
<?php
declare(strict_types=1);
namespace App\Service;
use Hyperf\DbConnection\Db;
/**
* TimescaleDB 连续聚合刷新实现
*
* 调用 PG 的 refresh_continuous_aggregate 存储过程,按整日窗口刷新。
*/
class ContinuousAggregateRefresher implements AggregateRefresherInterface
{
public function refresh(string $view, string $refresh_date): void
{
$start = sprintf("'%s 00:00:00+00'::timestamptz", $refresh_date);
$end = sprintf("'%s 23:59:59.999999+00'::timestamptz", $refresh_date);
// view 在 P23.2 中硬编码为 'orders_daily_by_created',但仍走 PDO quote 防御
$quoted_view = Db::getPdo()->quote($view);
Db::statement(sprintf(
'CALL refresh_continuous_aggregate(%s, %s, %s)',
$quoted_view,
$start,
$end
));
}
}
@@ -0,0 +1,76 @@
<?php
declare(strict_types=1);
namespace App\Service;
use App\Model\AggregateRefreshQueue;
use App\Utils\Log;
use Carbon\Carbon;
use Throwable;
/**
* 订单聚合刷新任务
*
* 消费 aggregate_refresh_queue 中 created_at < now() - 1 hour 的项,
* 逐条调用 AggregateRefresherInterface 刷新对应日期的连续聚合,
* 成功后从队列中删除;失败保留队列项由下次任务重试。
*
* 由 OrderAggregatesRefreshCommandCLI 入口)和 Crontab(定时入口)共同调用。
*/
class OrderAggregatesRefreshJob
{
public function __construct(private AggregateRefresherInterface $refresher)
{
}
/**
* @return array{processed: int, failed: int} 处理与失败计数
*/
public function run(): array
{
$items = AggregateRefreshQueue::query()
->where('created_at', '<', Carbon::now()->subHour())
->orderBy('refresh_date')
->orderBy('aggregate_view')
->get();
if ($items->isEmpty()) {
Log::get()->info('orders:refresh-aggregates queue empty');
return ['processed' => 0, 'failed' => 0];
}
$processed = 0;
$failed = 0;
foreach ($items as $item) {
$view = $item->aggregate_view;
$date = $item->refresh_date;
try {
$this->refresher->refresh($view, $date);
AggregateRefreshQueue::query()
->where('refresh_date', $date)
->where('aggregate_view', $view)
->delete();
$processed++;
} catch (Throwable $e) {
Log::get()->error('orders:refresh-aggregates failed', [
'view' => $view,
'date' => $date,
'error' => $e->getMessage(),
]);
$failed++;
}
}
Log::get()->info('orders:refresh-aggregates done', [
'processed' => $processed,
'failed' => $failed,
]);
return ['processed' => $processed, 'failed' => $failed];
}
}
+958
View File
@@ -0,0 +1,958 @@
#!/bin/bash
# ============================================================================
# RabbitMQ 配置管理脚本
# 支持从数据库读取平台列表,自动配置 Exchange、Queue、Binding 和用户
# ============================================================================
set -e
# 脚本所在目录
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BACKEND_DIR="$(dirname "$SCRIPT_DIR")"
ENV_FILE="$BACKEND_DIR/.env"
MQ_USER_CONFIG="$BACKEND_DIR/config/autoload/mq_user.php"
# RabbitMQ 连接信息
RABBITMQ_HOST="127.0.0.1"
RABBITMQ_PORT="15672"
RABBITMQ_USER="admin"
RABBITMQ_PASS="admin"
VHOST="datahub"
# 数据类型列表
DATA_TYPES=("orders" "products" "refunds" "inventory")
# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# ============================================================================
# 帮助信息
# ============================================================================
show_help() {
# 使用 printf '%b' 解释 ANSI 转义序列
printf '%b\n' "$(cat << EOF
${BLUE}RabbitMQ 配置管理脚本${NC}
${YELLOW}用法:${NC}
$0 <command> [options]
${YELLOW}命令:${NC}
${GREEN}init${NC} 从数据库读取所有启用的平台,完全重建 MQ 配置
- 删除现有 VHost 及所有资源
- 从 platforms 表读取 enabled=true 的平台
- 创建所有 Exchange、Queue、Binding
- 生成随机密码并保存到 mq_user.php
${GREEN}add <platform>${NC} 添加单个平台的 MQ 配置
- 创建平台的 Exchange 和 Binding
- 创建平台用户并设置权限
- 生成随机密码并更新 mq_user.php
示例: $0 add shopee
${GREEN}remove <platform>${NC} 移除单个平台的 MQ 配置
- 删除平台的 Exchange 和 Binding
- 删除平台用户
- 从 mq_user.php 中移除配置
示例: $0 remove shopee
${GREEN}list${NC} 列出当前 MQ 中已配置的平台
${GREEN}version${NC} 显示 RabbitMQ 服务器版本信息
${GREEN}reset-password${NC} 重置指定用户的密码
--user <name> 用户名称 (consumer/ops/平台名)
示例: $0 reset-password --user consumer
示例: $0 reset-password --user tmall
${GREEN}--help, -h${NC} 显示此帮助信息
${YELLOW}配置文件:${NC}
环境配置: $ENV_FILE (包含 MQ_PASSWORD_* 密码变量)
用户配置: $MQ_USER_CONFIG (动态从数据库读取平台列表)
${YELLOW}平台名称转换规则:${NC}
- 空格转换为下划线
- 全部转为小写
- 示例: "Amazon Japan" -> "amazon_japan"
${YELLOW}数据库表结构:${NC}
platforms 表需要包含以下字段:
- id: 平台 ID
- name: 平台名称
- enabled: 是否启用 (boolean)
${YELLOW}示例:${NC}
$0 init # 完全重建 MQ 配置
$0 add lazada # 添加 lazada 平台
$0 remove kaola # 移除 kaola 平台
$0 list # 列出已配置的平台
EOF
)"
}
# ============================================================================
# 依赖检测
# ============================================================================
# 检测是否在容器环境中运行
is_container() {
# 检测 Docker
if [[ -f /.dockerenv ]]; then
return 0
fi
# 检测 Podman (通过 /run/.containerenv 文件)
if [[ -f /run/.containerenv ]]; then
return 0
fi
# 检测 cgroup (Docker/Kubernetes/Podman)
if grep -qE 'docker|kubepods|containerd|libpod' /proc/1/cgroup 2>/dev/null; then
return 0
fi
# 检测 Kubernetes
if [[ -n "${KUBERNETES_SERVICE_HOST:-}" ]]; then
return 0
fi
# 检测容器运行时环境变量
if [[ -n "${container:-}" ]]; then
return 0
fi
return 1
}
check_dependencies() {
local missing=()
local in_container=false
if is_container; then
in_container=true
fi
# 检查 rabbitmqadmin
if ! command -v rabbitmqadmin &> /dev/null; then
missing+=("rabbitmqadmin")
fi
# 检查 php 及必要扩展
if ! command -v php &> /dev/null; then
missing+=("php")
else
# 检查 PHP 扩展
if ! php -m 2>/dev/null | grep -qi "^pdo_pgsql$"; then
missing+=("php-pdo_pgsql")
fi
if ! php -m 2>/dev/null | grep -qi "^sodium$"; then
missing+=("php-sodium")
fi
fi
if [[ ${#missing[@]} -gt 0 ]]; then
echo -e "${RED}错误: 缺少必要的依赖工具${NC}"
echo ""
if $in_container; then
echo -e "${YELLOW}检测到容器环境 (Docker/Podman/Kubernetes)${NC}"
echo "建议在 Dockerfile/Containerfile 或镜像中预装以下工具:"
echo ""
fi
for tool in "${missing[@]}"; do
case "$tool" in
rabbitmqadmin)
echo -e "${YELLOW}rabbitmqadmin${NC} - RabbitMQ 管理工具"
if $in_container; then
echo " Dockerfile/Containerfile 示例:"
echo " RUN wget -O /usr/local/bin/rabbitmqadmin \\"
echo " http://rabbitmq:15672/cli/rabbitmqadmin && \\"
echo " chmod +x /usr/local/bin/rabbitmqadmin"
echo " 或使用 pip:"
echo " RUN pip install rabbitmqadmin"
else
echo " 安装方式:"
echo " 1. 从 RabbitMQ 管理界面下载:"
echo " wget http://localhost:15672/cli/rabbitmqadmin"
echo " chmod +x rabbitmqadmin"
echo " sudo mv rabbitmqadmin /usr/local/bin/"
echo " 2. 或者使用 pip:"
echo " pip install rabbitmqadmin"
fi
echo ""
;;
php)
echo -e "${YELLOW}php${NC} - PHP 命令行"
if $in_container; then
echo " 如果使用 PHP 基础镜像,php 应该已经存在"
echo " 否则参考: https://hub.docker.com/_/php"
else
echo " 安装方式:"
echo " Ubuntu/Debian: sudo apt install php-cli"
echo " CentOS/RHEL: sudo yum install php-cli"
echo " macOS: brew install php"
fi
echo ""
;;
php-pdo_pgsql)
echo -e "${YELLOW}php-pdo_pgsql${NC} - PHP PostgreSQL PDO 扩展"
if $in_container; then
echo " Dockerfile/Containerfile 示例:"
echo " RUN docker-php-ext-install pdo_pgsql"
else
echo " 安装方式:"
echo " Ubuntu/Debian: sudo apt install php-pgsql"
echo " CentOS/RHEL: sudo yum install php-pgsql"
fi
echo ""
;;
php-sodium)
echo -e "${YELLOW}php-sodium${NC} - PHP Sodium 扩展"
if $in_container; then
echo " Dockerfile/Containerfile 示例:"
echo " RUN docker-php-ext-install sodium"
else
echo " 安装方式:"
echo " Ubuntu/Debian: sudo apt install php-sodium"
echo " CentOS/RHEL: sudo yum install php-sodium"
echo " 注意: PHP 7.2+ 默认包含 sodium 扩展"
fi
echo ""
;;
esac
done
exit 1
fi
}
# ============================================================================
# 工具函数
# ============================================================================
# 输出信息
info() {
echo -e "${GREEN}${NC} $1"
}
warn() {
echo -e "${YELLOW}${NC} $1"
}
error() {
echo -e "${RED}${NC} $1"
exit 1
}
# 从 .env 文件读取配置
load_env() {
if [[ ! -f "$ENV_FILE" ]]; then
error "找不到 .env 文件: $ENV_FILE"
fi
# 读取数据库配置
DB_HOST=$(grep -E "^DB_HOST=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_PORT=$(grep -E "^DB_PORT=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_DATABASE=$(grep -E "^DB_DATABASE=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_USERNAME=$(grep -E "^DB_USERNAME=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_PASSWORD=$(grep -E "^DB_PASSWORD=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
if [[ -z "$DB_HOST" || -z "$DB_DATABASE" ]]; then
error "无法从 .env 文件读取数据库配置"
fi
}
# 平台名称标准化:空格转下划线,全部小写
normalize_platform_name() {
echo "$1" | tr '[:upper:]' '[:lower:]' | tr ' ' '_'
}
# 生成随机密码 (16位,使用 PHP sodium)
generate_password() {
php -r "echo bin2hex(sodium_crypto_secretbox_keygen());" | head -c 16
}
# 从数据库获取所有启用的平台 (使用 PHP PDO)
get_platforms_from_db() {
load_env
php << EOF
<?php
try {
\$dsn = "pgsql:host=$DB_HOST;port=$DB_PORT;dbname=$DB_DATABASE";
\$pdo = new PDO(\$dsn, '$DB_USERNAME', '$DB_PASSWORD', [
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION
]);
\$stmt = \$pdo->query("SELECT name FROM platforms WHERE enabled = true ORDER BY id");
while (\$row = \$stmt->fetch(PDO::FETCH_ASSOC)) {
echo \$row['name'] . "\n";
}
} catch (PDOException \$e) {
fwrite(STDERR, "数据库连接失败: " . \$e->getMessage() . "\n");
exit(1);
}
EOF
}
# 检查平台是否在数据库中存在 (使用 PHP PDO)
platform_exists_in_db() {
local platform_name="$1"
load_env
local count=$(php << EOF
<?php
try {
\$dsn = "pgsql:host=$DB_HOST;port=$DB_PORT;dbname=$DB_DATABASE";
\$pdo = new PDO(\$dsn, '$DB_USERNAME', '$DB_PASSWORD', [
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION
]);
\$stmt = \$pdo->prepare("SELECT COUNT(*) FROM platforms WHERE LOWER(REPLACE(name, ' ', '_')) = LOWER(:name)");
\$stmt->execute(['name' => '$platform_name']);
echo \$stmt->fetchColumn();
} catch (PDOException \$e) {
echo "0";
}
EOF
)
[[ "$count" -gt 0 ]]
}
# ============================================================================
# .env 密码配置操作
# ============================================================================
# 添加或更新 .env 中的密码变量
# 用法: add_password_to_env "SHOPEE" "password123"
add_password_to_env() {
local platform_upper="$1"
local password="$2"
local env_key="MQ_PASSWORD_${platform_upper}"
# 检查变量是否已存在
if grep -q "^${env_key}=" "$ENV_FILE" 2>/dev/null; then
# 更新现有变量
sed -i "s|^${env_key}=.*|${env_key}=\"${password}\"|" "$ENV_FILE"
else
# 检查是否需要添加分隔注释
if ! grep -q "^# RabbitMQ" "$ENV_FILE" 2>/dev/null; then
echo "" >> "$ENV_FILE"
echo "# RabbitMQ 用户密码配置 (由 bin/rabbitmq.sh 自动生成)" >> "$ENV_FILE"
fi
# 追加新变量
echo "${env_key}=\"${password}\"" >> "$ENV_FILE"
fi
}
# 从 .env 中移除密码变量
remove_password_from_env() {
local platform_upper="$1"
local env_key="MQ_PASSWORD_${platform_upper}"
if grep -q "^${env_key}=" "$ENV_FILE" 2>/dev/null; then
sed -i "/^${env_key}=/d" "$ENV_FILE"
fi
}
# 读取 .env 中的现有密码
get_existing_password_from_env() {
local platform_upper="$1"
local env_key="MQ_PASSWORD_${platform_upper}"
grep "^${env_key}=" "$ENV_FILE" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'"
}
# 批量写入所有密码到 .env (用于 init 模式)
write_all_passwords_to_env() {
local -n platforms_ref=$1
local -n passwords_ref=$2
local consumer_password="$3"
local ops_password="$4"
# 先清理现有的 MQ_PASSWORD_ 变量
sed -i '/^MQ_PASSWORD_/d' "$ENV_FILE" 2>/dev/null || true
sed -i '/^# RabbitMQ 用户密码配置/d' "$ENV_FILE" 2>/dev/null || true
# 移除文件末尾的多余空行
sed -i -e :a -e '/^\n*$/{$d;N;ba' -e '}' "$ENV_FILE" 2>/dev/null || true
# 追加新配置
{
echo ""
echo "# RabbitMQ 用户密码配置 (由 bin/rabbitmq.sh 自动生成,时间: $(date '+%Y-%m-%d %H:%M:%S'))"
echo "MQ_PASSWORD_CONSUMER=\"${consumer_password}\""
echo "MQ_PASSWORD_OPS=\"${ops_password}\""
for platform in "${platforms_ref[@]}"; do
local platform_upper=$(echo "$platform" | tr '[:lower:]' '[:upper:]')
echo "MQ_PASSWORD_${platform_upper}=\"${passwords_ref[$platform]}\""
done
} >> "$ENV_FILE"
}
# ============================================================================
# RabbitMQ 操作函数
# ============================================================================
# 创建单个平台的 MQ 配置
create_platform_config() {
local platform="$1"
local password="$2"
echo ""
echo -e "${BLUE}处理平台: ${platform}${NC}"
# 创建平台业务 Exchange
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "${platform}.exchange" --vhost "$VHOST" \
--type topic --durable true
info "创建业务 Exchange: ${platform}.exchange"
# 绑定到业务队列
for dtype in "${DATA_TYPES[@]}"; do
local dtype_singular="${dtype%s}"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "${platform}.exchange" --destination "${dtype}.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "${dtype_singular}.${platform}"
done
info "绑定到业务队列 (${#DATA_TYPES[@]}个)"
# 创建平台错误 Exchange
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "${platform}.errors.exchange" --vhost "$VHOST" \
--type topic --durable true
info "创建错误 Exchange: ${platform}.errors.exchange"
# 绑定到错误队列
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "${platform}.errors.exchange" --destination "errors.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "#"
info "绑定到错误队列"
# 创建平台用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "user_${platform}" --password "$password" --tags ""
info "创建用户: user_${platform}"
# 配置平台用户权限
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare permissions --vhost "$VHOST" --user "user_${platform}" \
--configure "^${platform}\\.(exchange|errors\\.exchange)$" \
--write "^${platform}\\.(exchange|errors\\.exchange)$" \
--read "^${platform}\\.errors\\..*$"
info "配置用户权限"
}
# 删除单个平台的 MQ 配置
remove_platform_config() {
local platform="$1"
echo ""
echo -e "${BLUE}移除平台: ${platform}${NC}"
# 删除用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete user --name "user_${platform}" 2>/dev/null || true
info "删除用户: user_${platform}"
# 删除 Exchange (会自动删除相关的 Binding)
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete exchange --name "${platform}.exchange" --vhost "$VHOST" 2>/dev/null || true
info "删除 Exchange: ${platform}.exchange"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete exchange --name "${platform}.errors.exchange" --vhost "$VHOST" 2>/dev/null || true
info "删除 Exchange: ${platform}.errors.exchange"
}
# 创建基础设施 (VHost, 主 Exchange, DLX, 队列等)
create_infrastructure() {
echo ""
echo "========================================"
echo "创建基础设施..."
echo "========================================"
# 1. 创建 VHost
echo ""
echo "创建 VHost: $VHOST"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare vhost --name "$VHOST"
info "VHost 创建成功"
# 2. 创建主 Exchange
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "main.exchange" --vhost "$VHOST" \
--type topic --durable true
info "创建主 Exchange: main.exchange"
# 3. 创建 DLX
echo ""
echo "创建死信交换机 (DLX)..."
for dtype in "${DATA_TYPES[@]}"; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "dlx.${dtype}" --vhost "$VHOST" \
--type topic --durable true
info "创建 DLX: dlx.${dtype}"
done
# 4. 创建主业务队列
echo ""
echo "创建主业务队列..."
for dtype in "${DATA_TYPES[@]}"; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare queue --name "${dtype}.queue" --vhost "$VHOST" --durable true \
--arguments "{\"x-message-ttl\":86400000,\"x-dead-letter-exchange\":\"dlx.${dtype}\",\"x-dead-letter-routing-key\":\"retry\"}"
info "创建队列: ${dtype}.queue"
done
# 5. 创建重试队列
echo ""
echo "创建重试队列..."
for dtype in "${DATA_TYPES[@]}"; do
local dtype_singular="${dtype%s}"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare queue --name "${dtype}.retry.queue" --vhost "$VHOST" --durable true \
--arguments "{\"x-message-ttl\":5000,\"x-dead-letter-exchange\":\"main.exchange\",\"x-dead-letter-routing-key\":\"${dtype_singular}.retry\"}"
info "创建重试队列: ${dtype}.retry.queue"
done
# 6. 创建错误 Exchange 和队列
echo ""
echo "创建错误 Exchange 和队列..."
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "errors.exchange" --vhost "$VHOST" \
--type topic --durable true
info "创建错误 Exchange: errors.exchange"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare queue --name "errors.queue" --vhost "$VHOST" --durable true \
--arguments '{"x-message-ttl":604800000}'
info "创建错误队列: errors.queue"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "errors.exchange" --destination "errors.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "#"
info "绑定: errors.exchange → errors.queue"
# 7. 绑定主 Exchange 到主队列
echo ""
echo "绑定主 Exchange 到主队列..."
for dtype in "${DATA_TYPES[@]}"; do
local dtype_singular="${dtype%s}"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "main.exchange" --destination "${dtype}.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "${dtype_singular}.#"
info "绑定: main.exchange → ${dtype}.queue (routing_key: ${dtype_singular}.#)"
done
# 8. 绑定 DLX 到重试队列
echo ""
echo "绑定 DLX 到重试队列..."
for dtype in "${DATA_TYPES[@]}"; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "dlx.${dtype}" --destination "${dtype}.retry.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "retry"
info "绑定: dlx.${dtype}${dtype}.retry.queue"
done
}
# 创建系统用户 (consumer 和 ops)
create_system_users() {
local consumer_password="$1"
local ops_password="$2"
echo ""
echo "========================================"
echo "创建系统用户..."
echo "========================================"
# 创建 datahub 消费者用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "user_datahub_consumer" --password "$consumer_password" --tags ""
info "创建用户: user_datahub_consumer"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare permissions --vhost "$VHOST" --user "user_datahub_consumer" \
--configure "^(main\\.exchange|errors\\.exchange|dlx\\..*)|(.*\\.queue)$" \
--write "^(orders|products|refunds|inventory).*\\.queue$|(dlx\\..*)|(errors\\.exchange)|(.*\\.errors\\.exchange)$" \
--read "^(main\\.exchange|(orders|products|refunds|inventory).*\\.queue|dlx\\..*)$"
info "配置 consumer 用户权限"
# 创建运维监控用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "user_ops" --password "$ops_password" --tags ""
info "创建用户: user_ops"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare permissions --vhost "$VHOST" --user "user_ops" \
--configure "^errors\\..*$" --write "" --read "^errors\\.queue$"
info "配置 ops 用户权限"
}
# 清理现有配置
cleanup_existing() {
echo "========================================"
echo "开始清理现有配置..."
echo "========================================"
# 检查 VHost 是否存在
echo ""
echo "检查 VHost: $VHOST 是否存在..."
VHOST_EXISTS=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list vhosts | grep -w "$VHOST" || true)
if [[ -n "$VHOST_EXISTS" ]]; then
warn "VHost '$VHOST' 已存在,将删除所有现有配置..."
# 获取所有平台用户并删除
local users=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list users name -f tsv 2>/dev/null | grep "^user_" || true)
for user in $users; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete user --name "$user" 2>/dev/null || true
info "删除用户: $user"
done
# 删除 VHost
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete vhost --name "$VHOST"
info "VHost '$VHOST' 及其所有资源已删除"
else
echo "VHost '$VHOST' 不存在"
fi
}
# ============================================================================
# 主命令函数
# ============================================================================
# init 命令:完全重建
cmd_init() {
echo -e "${BLUE}========================================"
echo "RabbitMQ 初始化模式"
echo "========================================${NC}"
# 1. 从数据库获取平台列表
echo ""
echo "从数据库读取平台列表..."
local db_platforms=$(get_platforms_from_db)
if [[ -z "$db_platforms" ]]; then
error "无法从数据库获取平台列表,或没有启用的平台"
fi
# 转换平台名称
declare -a PLATFORMS=()
declare -A PLATFORM_PASSWORDS=()
while IFS= read -r platform; do
local normalized=$(normalize_platform_name "$platform")
PLATFORMS+=("$normalized")
PLATFORM_PASSWORDS["$normalized"]=$(generate_password)
done <<< "$db_platforms"
echo "找到 ${#PLATFORMS[@]} 个启用的平台:"
printf " - %s\n" "${PLATFORMS[@]}"
# 2. 清理现有配置
cleanup_existing
# 3. 创建基础设施
create_infrastructure
# 4. 生成系统用户密码
local consumer_password=$(generate_password)
local ops_password=$(generate_password)
# 5. 创建系统用户
create_system_users "$consumer_password" "$ops_password"
# 6. 为每个平台创建配置
echo ""
echo "========================================"
echo "创建平台配置..."
echo "========================================"
for platform in "${PLATFORMS[@]}"; do
create_platform_config "$platform" "${PLATFORM_PASSWORDS[$platform]}"
done
# 7. 写入密码到 .env
echo ""
echo "========================================"
echo "写入密码到 .env..."
echo "========================================"
write_all_passwords_to_env PLATFORMS PLATFORM_PASSWORDS "$consumer_password" "$ops_password"
info "密码已写入: $ENV_FILE"
# 8. 输出摘要
echo ""
echo -e "${GREEN}========================================"
echo "RabbitMQ 配置完成!"
echo "========================================${NC}"
echo ""
echo "已创建:"
echo "- 1 个 VHost: $VHOST"
echo "- 1 个主 Exchange: main.exchange"
echo "- 4 个 DLX: dlx.orders, dlx.products, dlx.refunds, dlx.inventory"
echo "- 4 个主队列: orders.queue, products.queue, refunds.queue, inventory.queue"
echo "- 4 个重试队列: *.retry.queue"
echo "- 1 个错误队列: errors.queue"
echo "- ${#PLATFORMS[@]} 个平台配置"
echo "- $((${#PLATFORMS[@]} + 2)) 个用户"
echo ""
echo "密码配置: $ENV_FILE (MQ_PASSWORD_* 变量)"
echo "用户配置: $MQ_USER_CONFIG (动态从数据库读取)"
}
# add 命令:添加单个平台
cmd_add() {
local platform_input="$1"
if [[ -z "$platform_input" ]]; then
error "请指定要添加的平台名称,例如: $0 add shopee"
fi
local platform=$(normalize_platform_name "$platform_input")
echo -e "${BLUE}========================================"
echo "添加平台: $platform"
echo "========================================${NC}"
# 检查 VHost 是否存在
VHOST_EXISTS=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list vhosts | grep -w "$VHOST" || true)
if [[ -z "$VHOST_EXISTS" ]]; then
error "VHost '$VHOST' 不存在,请先运行 '$0 init' 初始化"
fi
# 检查平台是否已存在
local exchange_exists=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list exchanges --vhost "$VHOST" | grep -w "${platform}.exchange" || true)
if [[ -n "$exchange_exists" ]]; then
error "平台 '$platform' 已存在"
fi
# 生成密码
local password=$(generate_password)
# 创建平台配置
create_platform_config "$platform" "$password"
# 写入密码到 .env
local platform_upper=$(echo "$platform" | tr '[:lower:]' '[:upper:]')
add_password_to_env "$platform_upper" "$password"
echo ""
echo -e "${GREEN}========================================"
echo "平台 $platform 添加成功!"
echo "========================================${NC}"
echo ""
echo "用户: user_${platform}"
echo "密码已写入 .env: MQ_PASSWORD_${platform_upper}"
}
# remove 命令:移除单个平台
cmd_remove() {
local platform_input="$1"
if [[ -z "$platform_input" ]]; then
error "请指定要移除的平台名称,例如: $0 remove shopee"
fi
local platform=$(normalize_platform_name "$platform_input")
echo -e "${BLUE}========================================"
echo "移除平台: $platform"
echo "========================================${NC}"
# 移除 MQ 配置
remove_platform_config "$platform"
# 从 .env 中移除密码
local platform_upper=$(echo "$platform" | tr '[:lower:]' '[:upper:]')
remove_password_from_env "$platform_upper"
echo ""
echo -e "${GREEN}========================================"
echo "平台 $platform 移除成功!"
echo "========================================${NC}"
echo ""
echo "已从 .env 移除: MQ_PASSWORD_${platform_upper}"
}
# reset-password 命令:重置用户密码
cmd_reset_password() {
local user_name=""
# 解析参数
while [[ $# -gt 0 ]]; do
case "$1" in
--user)
user_name="$2"
shift 2
;;
*)
error "未知参数: $1\n用法: $0 reset-password --user <name>"
;;
esac
done
if [[ -z "$user_name" ]]; then
error "请指定用户名称\n用法: $0 reset-password --user <name>\n示例: $0 reset-password --user consumer"
fi
# 标准化用户名称
local normalized_name=$(normalize_platform_name "$user_name")
local new_password=$(generate_password)
echo -e "${BLUE}========================================"
echo "重置用户密码: $normalized_name"
echo "========================================${NC}"
# 判断用户类型并处理
case "$normalized_name" in
consumer)
local mq_user="user_datahub_consumer"
local env_key="CONSUMER"
;;
ops)
local mq_user="user_ops"
local env_key="OPS"
;;
*)
# 平台用户
local mq_user="user_${normalized_name}"
local env_key=$(echo "$normalized_name" | tr '[:lower:]' '[:upper:]')
;;
esac
# 检查用户是否存在
local user_exists=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list users 2>/dev/null | grep -w "${mq_user}" || true)
if [[ -z "$user_exists" ]]; then
error "用户 '$mq_user' 不存在"
fi
# 更新 RabbitMQ 用户密码
echo ""
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "$mq_user" --password "$new_password" --tags ""
info "RabbitMQ 用户密码已更新: $mq_user"
# 更新 .env 文件
add_password_to_env "$env_key" "$new_password"
info "密码已写入 .env: MQ_PASSWORD_${env_key}"
echo ""
echo -e "${GREEN}========================================"
echo "密码重置成功!"
echo "========================================${NC}"
echo ""
echo "用户: $mq_user"
echo "新密码已保存到 .env: MQ_PASSWORD_${env_key}"
echo ""
echo -e "${YELLOW}注意: 请确保相关服务使用新密码重新连接${NC}"
}
# list 命令:列出已配置的平台
cmd_list() {
echo -e "${BLUE}========================================"
echo "已配置的平台列表"
echo "========================================${NC}"
# 检查 VHost 是否存在
VHOST_EXISTS=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list vhosts 2>/dev/null | grep -w "$VHOST" || true)
if [[ -z "$VHOST_EXISTS" ]]; then
warn "VHost '$VHOST' 不存在"
return
fi
# 获取所有平台 Exchange (排除系统 Exchange)
echo ""
echo "MQ 中的平台:"
local exchanges=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list exchanges --vhost "$VHOST" name -f tsv 2>/dev/null | grep -E "^[a-z0-9_]+\.exchange$" | grep -v "^main\." | grep -v "^errors\." | sed 's/\.exchange$//' | sort)
if [[ -n "$exchanges" ]]; then
echo "$exchanges" | while read platform; do
echo " - $platform"
done
else
echo " (无)"
fi
# 显示配置文件中的平台
if [[ -f "$MQ_USER_CONFIG" ]]; then
echo ""
echo "配置文件中的平台:"
php -r "
\$config = include '$MQ_USER_CONFIG';
foreach (\$config['platforms'] ?? [] as \$name => \$data) {
echo \" - \$name\n\";
}
" 2>/dev/null || echo " (无法读取配置文件)"
fi
}
# version 命令:显示 RabbitMQ 服务器版本信息
cmd_version() {
echo -e "${BLUE}========================================"
echo -e "RabbitMQ 服务器版本信息"
echo -e "========================================${NC}"
echo ""
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
show overview 2>/dev/null || error "无法连接到 RabbitMQ 服务器"
}
# ============================================================================
# 主入口
# ============================================================================
main() {
local command="${1:-}"
# 显示帮助不需要检测依赖
if [[ "$command" != "--help" && "$command" != "-h" && "$command" != "help" && -n "$command" ]]; then
check_dependencies
fi
case "$command" in
init)
cmd_init
;;
add)
cmd_add "$2"
;;
remove)
cmd_remove "$2"
;;
list)
cmd_list
;;
version)
cmd_version
;;
reset-password)
shift
cmd_reset_password "$@"
;;
--help|-h|help|"")
show_help
;;
*)
error "未知命令: $command\n使用 '$0 --help' 查看帮助"
;;
esac
}
main "$@"
+327 -137
View File
@@ -1,25 +1,36 @@
#!/bin/bash
# ============================================================================
# RabbitMQ 配置管理脚本
# RabbitMQ 配置管理脚本 (适配 datahub-backend 容器环境)
# 支持从数据库读取平台列表,自动配置 Exchange、Queue、Binding 和用户
# 通过 RabbitMQ Management HTTP API (curl) 操作,无需 rabbitmqadmin
# ============================================================================
set -e
# 脚本所在目录
# 脚本所在目录 (容器内路径: /opt/www)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BACKEND_DIR="$(dirname "$SCRIPT_DIR")"
ENV_FILE="$BACKEND_DIR/.env"
MQ_USER_CONFIG="$BACKEND_DIR/config/autoload/mq_user.php"
# RabbitMQ 连接信息
RABBITMQ_HOST="127.0.0.1"
# .env 密码文件输出路径
# 容器内 /mnt/share 映射到宿主机 /var/container/share
# 执行完成后需手动合并到宿主机 /opt/datahub/backend/.env
ENV_FILE="/mnt/share/rabbitmq_passwords.env"
# RabbitMQ 连接信息 (容器间通过容器名通信)
RABBITMQ_HOST="datahub-rabbitmq"
RABBITMQ_PORT="15672"
RABBITMQ_USER="admin"
RABBITMQ_PASS="admin"
RABBITMQ_USER="user"
RABBITMQ_PASS="hub123456"
VHOST="datahub"
# URL 编码的 VHost (用于 HTTP API 路径)
VHOST_ENCODED=$(php -r "echo rawurlencode('$VHOST');" 2>/dev/null || echo "$VHOST")
# RabbitMQ API 基础 URL
MQ_API="http://${RABBITMQ_HOST}:${RABBITMQ_PORT}/api"
# 数据类型列表
DATA_TYPES=("orders" "products" "refunds" "inventory")
@@ -36,7 +47,7 @@ NC='\033[0m' # No Color
show_help() {
# 使用 printf '%b' 解释 ANSI 转义序列
printf '%b\n' "$(cat << EOF
${BLUE}RabbitMQ 配置管理脚本${NC}
${BLUE}RabbitMQ 配置管理脚本 (容器版)${NC}
${YELLOW}用法:${NC}
$0 <command> [options]
@@ -72,7 +83,7 @@ ${YELLOW}命令:${NC}
${GREEN}--help, -h${NC} 显示此帮助信息
${YELLOW}配置文件:${NC}
环境配置: $ENV_FILE (包含 MQ_PASSWORD_* 密码变量)
密码输出: /mnt/share/rabbitmq_passwords.env (宿主机 /var/container/share/)
用户配置: $MQ_USER_CONFIG (动态从数据库读取平台列表)
${YELLOW}平台名称转换规则:${NC}
@@ -138,9 +149,9 @@ check_dependencies() {
in_container=true
fi
# 检查 rabbitmqadmin
if ! command -v rabbitmqadmin &> /dev/null; then
missing+=("rabbitmqadmin")
# 检查 curl (用于调用 RabbitMQ HTTP API)
if ! command -v curl &> /dev/null; then
missing+=("curl")
fi
# 检查 php 及必要扩展
@@ -168,23 +179,15 @@ check_dependencies() {
for tool in "${missing[@]}"; do
case "$tool" in
rabbitmqadmin)
echo -e "${YELLOW}rabbitmqadmin${NC} - RabbitMQ 管理工具"
curl)
echo -e "${YELLOW}curl${NC} - HTTP 请求工具 (用于调用 RabbitMQ Management API)"
if $in_container; then
echo " Dockerfile/Containerfile 示例:"
echo " RUN wget -O /usr/local/bin/rabbitmqadmin \\"
echo " http://rabbitmq:15672/cli/rabbitmqadmin && \\"
echo " chmod +x /usr/local/bin/rabbitmqadmin"
echo " 或使用 pip:"
echo " RUN pip install rabbitmqadmin"
echo " Alpine: apk add curl"
echo " Debian: apt-get install curl"
else
echo " 安装方式:"
echo " 1. 从 RabbitMQ 管理界面下载:"
echo " wget http://localhost:15672/cli/rabbitmqadmin"
echo " chmod +x rabbitmqadmin"
echo " sudo mv rabbitmqadmin /usr/local/bin/"
echo " 2. 或者使用 pip:"
echo " pip install rabbitmqadmin"
echo " Ubuntu/Debian: sudo apt install curl"
echo " CentOS/RHEL: sudo yum install curl"
fi
echo ""
;;
@@ -222,7 +225,6 @@ check_dependencies() {
echo " 安装方式:"
echo " Ubuntu/Debian: sudo apt install php-sodium"
echo " CentOS/RHEL: sudo yum install php-sodium"
echo " 注意: PHP 7.2+ 默认包含 sodium 扩展"
fi
echo ""
;;
@@ -230,6 +232,160 @@ check_dependencies() {
done
exit 1
fi
# 检查 RabbitMQ Management API 是否可达
local http_code
http_code=$(curl -s -o /dev/null -w "%{http_code}" -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" \
"${MQ_API}/overview" 2>/dev/null || echo "000")
if [[ "$http_code" == "000" ]]; then
error "无法连接到 RabbitMQ Management API: ${MQ_API}\n请确认 datahub-rabbitmq 容器正在运行且 management 插件已启用"
elif [[ "$http_code" == "401" ]]; then
error "RabbitMQ 认证失败,请检查用户名和密码"
elif [[ "$http_code" != "200" ]]; then
error "RabbitMQ Management API 返回异常状态码: ${http_code}"
fi
}
# ============================================================================
# RabbitMQ HTTP API 封装函数
# ============================================================================
# 通用 API 请求函数
# 用法: mq_api_call <METHOD> <PATH> [JSON_DATA]
# 写操作 (PUT/POST/DELETE) 自动丢弃响应体,GET 保留输出
mq_api_call() {
local method="$1"
local path="$2"
local data="${3:-}"
if [[ "$method" != "GET" ]]; then
if [[ -n "$data" ]]; then
curl -s -o /dev/null -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" \
-H "content-type: application/json" \
-X "$method" \
-d "$data" \
"${MQ_API}${path}"
else
curl -s -o /dev/null -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" \
-H "content-type: application/json" \
-X "$method" \
"${MQ_API}${path}"
fi
else
if [[ -n "$data" ]]; then
curl -s -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" \
-H "content-type: application/json" \
-X "$method" \
-d "$data" \
"${MQ_API}${path}"
else
curl -s -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" \
-H "content-type: application/json" \
-X "$method" \
"${MQ_API}${path}"
fi
fi
}
# 声明 VHost
mq_declare_vhost() {
local name="$1"
local encoded
encoded=$(php -r "echo rawurlencode('$name');" 2>/dev/null || echo "$name")
mq_api_call PUT "/vhosts/${encoded}" '{}'
}
# 删除 VHost
mq_delete_vhost() {
local name="$1"
local encoded
encoded=$(php -r "echo rawurlencode('$name');" 2>/dev/null || echo "$name")
mq_api_call DELETE "/vhosts/${encoded}" 2>/dev/null || true
}
# 列出 VHosts
mq_list_vhosts() {
mq_api_call GET "/vhosts" 2>/dev/null
}
# 声明 Exchange
mq_declare_exchange() {
local name="$1"
local type="${2:-topic}"
local durable="${3:-true}"
local body
body=$(printf '{"type":"%s","durable":%s}' "$type" "$durable")
mq_api_call PUT "/exchanges/${VHOST_ENCODED}/${name}" "$body"
}
# 删除 Exchange
mq_delete_exchange() {
local name="$1"
mq_api_call DELETE "/exchanges/${VHOST_ENCODED}/${name}" 2>/dev/null || true
}
# 声明 Queue
mq_declare_queue() {
local name="$1"
local arguments="$2"
if [[ -z "$arguments" ]]; then
arguments='{}'
fi
local body
body=$(printf '{"durable":true,"arguments":%s}' "$arguments")
mq_api_call PUT "/queues/${VHOST_ENCODED}/${name}" "$body"
}
# 声明 Binding
mq_declare_binding() {
local source="$1"
local destination="$2"
local routing_key="$3"
local dest_type="${4:-queue}"
local dest_char="q"
if [[ "$dest_type" == "exchange" ]]; then
dest_char="e"
fi
mq_api_call POST "/bindings/${VHOST_ENCODED}/e/${source}/${dest_char}/${destination}" \
"$(printf '{"routing_key":"%s"}' "$routing_key")"
}
# 声明用户
mq_declare_user() {
local name="$1"
local password="$2"
local tags="${3:-}"
mq_api_call PUT "/users/${name}" \
"$(printf '{"password":"%s","tags":"%s"}' "$password" "$tags")"
}
# 删除用户
mq_delete_user() {
local name="$1"
mq_api_call DELETE "/users/${name}" 2>/dev/null || true
}
# 列出用户
mq_list_users() {
mq_api_call GET "/users" 2>/dev/null
}
# 设置用户权限
mq_set_permissions() {
local user="$1"
local configure="$2"
local write="$3"
local read="$4"
mq_api_call PUT "/permissions/${VHOST_ENCODED}/${user}" \
"$(printf '{"configure":"%s","write":"%s","read":"%s"}' "$configure" "$write" "$read")"
}
# 列出 Exchanges
mq_list_exchanges() {
mq_api_call GET "/exchanges/${VHOST_ENCODED}" 2>/dev/null
}
# ============================================================================
@@ -250,21 +406,17 @@ error() {
exit 1
}
# 从 .env 文件读取配置
# 从容器环境变量读取数据库配置
load_env() {
if [[ ! -f "$ENV_FILE" ]]; then
error "找不到 .env 文件: $ENV_FILE"
fi
# 读取数据库配置
DB_HOST=$(grep -E "^DB_HOST=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_PORT=$(grep -E "^DB_PORT=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_DATABASE=$(grep -E "^DB_DATABASE=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_USERNAME=$(grep -E "^DB_USERNAME=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
DB_PASSWORD=$(grep -E "^DB_PASSWORD=" "$ENV_FILE" | cut -d'=' -f2 | tr -d '"' | tr -d "'")
# 容器内通过环境变量注入,直接读取
DB_HOST="${DB_HOST:-}"
DB_PORT="${DB_PORT:-5432}"
DB_DATABASE="${DB_DATABASE:-}"
DB_USERNAME="${DB_USERNAME:-}"
DB_PASSWORD="${DB_PASSWORD:-}"
if [[ -z "$DB_HOST" || -z "$DB_DATABASE" ]]; then
error "无法从 .env 文件读取数据库配置"
error "无法读取数据库配置,请确认容器环境变量 DB_HOST, DB_DATABASE 等已设置"
fi
}
@@ -405,43 +557,33 @@ create_platform_config() {
echo -e "${BLUE}处理平台: ${platform}${NC}"
# 创建平台业务 Exchange
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "${platform}.exchange" --vhost "$VHOST" \
--type topic --durable true
mq_declare_exchange "${platform}.exchange" "topic" "true"
info "创建业务 Exchange: ${platform}.exchange"
# 绑定到业务队列
for dtype in "${DATA_TYPES[@]}"; do
local dtype_singular="${dtype%s}"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "${platform}.exchange" --destination "${dtype}.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "${dtype_singular}.${platform}"
mq_declare_binding "${platform}.exchange" "${dtype}.queue" "${dtype_singular}.${platform}"
done
info "绑定到业务队列 (${#DATA_TYPES[@]}个)"
# 创建平台错误 Exchange
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "${platform}.errors.exchange" --vhost "$VHOST" \
--type topic --durable true
mq_declare_exchange "${platform}.errors.exchange" "topic" "true"
info "创建错误 Exchange: ${platform}.errors.exchange"
# 绑定到错误队列
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "${platform}.errors.exchange" --destination "errors.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "#"
mq_declare_binding "${platform}.errors.exchange" "errors.queue" "#"
info "绑定到错误队列"
# 创建平台用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "user_${platform}" --password "$password" --tags ""
mq_declare_user "user_${platform}" "$password" ""
info "创建用户: user_${platform}"
# 配置平台用户权限
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare permissions --vhost "$VHOST" --user "user_${platform}" \
--configure "^${platform}\\.(exchange|errors\\.exchange)$" \
--write "^${platform}\\.(exchange|errors\\.exchange)$" \
--read "^${platform}\\.errors\\..*$"
mq_set_permissions "user_${platform}" \
"^${platform}\\.(exchange|errors\\.exchange)$" \
"^${platform}\\.(exchange|errors\\.exchange)$" \
"^${platform}\\.errors\\..*$"
info "配置用户权限"
}
@@ -453,17 +595,14 @@ remove_platform_config() {
echo -e "${BLUE}移除平台: ${platform}${NC}"
# 删除用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete user --name "user_${platform}" 2>/dev/null || true
mq_delete_user "user_${platform}"
info "删除用户: user_${platform}"
# 删除 Exchange (会自动删除相关的 Binding)
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete exchange --name "${platform}.exchange" --vhost "$VHOST" 2>/dev/null || true
mq_delete_exchange "${platform}.exchange"
info "删除 Exchange: ${platform}.exchange"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete exchange --name "${platform}.errors.exchange" --vhost "$VHOST" 2>/dev/null || true
mq_delete_exchange "${platform}.errors.exchange"
info "删除 Exchange: ${platform}.errors.exchange"
}
@@ -477,23 +616,18 @@ create_infrastructure() {
# 1. 创建 VHost
echo ""
echo "创建 VHost: $VHOST"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare vhost --name "$VHOST"
mq_declare_vhost "$VHOST"
info "VHost 创建成功"
# 2. 创建主 Exchange
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "main.exchange" --vhost "$VHOST" \
--type topic --durable true
mq_declare_exchange "main.exchange" "topic" "true"
info "创建主 Exchange: main.exchange"
# 3. 创建 DLX
echo ""
echo "创建死信交换机 (DLX)..."
for dtype in "${DATA_TYPES[@]}"; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "dlx.${dtype}" --vhost "$VHOST" \
--type topic --durable true
mq_declare_exchange "dlx.${dtype}" "topic" "true"
info "创建 DLX: dlx.${dtype}"
done
@@ -501,9 +635,8 @@ create_infrastructure() {
echo ""
echo "创建主业务队列..."
for dtype in "${DATA_TYPES[@]}"; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare queue --name "${dtype}.queue" --vhost "$VHOST" --durable true \
--arguments "{\"x-message-ttl\":86400000,\"x-dead-letter-exchange\":\"dlx.${dtype}\",\"x-dead-letter-routing-key\":\"retry\"}"
mq_declare_queue "${dtype}.queue" \
'{"x-message-ttl":86400000,"x-dead-letter-exchange":"dlx.'"${dtype}"'","x-dead-letter-routing-key":"retry"}'
info "创建队列: ${dtype}.queue"
done
@@ -512,28 +645,21 @@ create_infrastructure() {
echo "创建重试队列..."
for dtype in "${DATA_TYPES[@]}"; do
local dtype_singular="${dtype%s}"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare queue --name "${dtype}.retry.queue" --vhost "$VHOST" --durable true \
--arguments "{\"x-message-ttl\":5000,\"x-dead-letter-exchange\":\"main.exchange\",\"x-dead-letter-routing-key\":\"${dtype_singular}.retry\"}"
mq_declare_queue "${dtype}.retry.queue" \
'{"x-message-ttl":5000,"x-dead-letter-exchange":"main.exchange","x-dead-letter-routing-key":"'"${dtype_singular}"'.retry"}'
info "创建重试队列: ${dtype}.retry.queue"
done
# 6. 创建错误 Exchange 和队列
echo ""
echo "创建错误 Exchange 和队列..."
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare exchange --name "errors.exchange" --vhost "$VHOST" \
--type topic --durable true
mq_declare_exchange "errors.exchange" "topic" "true"
info "创建错误 Exchange: errors.exchange"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare queue --name "errors.queue" --vhost "$VHOST" --durable true \
--arguments '{"x-message-ttl":604800000}'
mq_declare_queue "errors.queue" '{"x-message-ttl":604800000}'
info "创建错误队列: errors.queue"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "errors.exchange" --destination "errors.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "#"
mq_declare_binding "errors.exchange" "errors.queue" "#"
info "绑定: errors.exchange → errors.queue"
# 7. 绑定主 Exchange 到主队列
@@ -541,9 +667,7 @@ create_infrastructure() {
echo "绑定主 Exchange 到主队列..."
for dtype in "${DATA_TYPES[@]}"; do
local dtype_singular="${dtype%s}"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "main.exchange" --destination "${dtype}.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "${dtype_singular}.#"
mq_declare_binding "main.exchange" "${dtype}.queue" "${dtype_singular}.#"
info "绑定: main.exchange → ${dtype}.queue (routing_key: ${dtype_singular}.#)"
done
@@ -551,9 +675,7 @@ create_infrastructure() {
echo ""
echo "绑定 DLX 到重试队列..."
for dtype in "${DATA_TYPES[@]}"; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare binding --source "dlx.${dtype}" --destination "${dtype}.retry.queue" \
--destination-type queue --vhost "$VHOST" --routing-key "retry"
mq_declare_binding "dlx.${dtype}" "${dtype}.retry.queue" "retry"
info "绑定: dlx.${dtype}${dtype}.retry.queue"
done
}
@@ -569,25 +691,21 @@ create_system_users() {
echo "========================================"
# 创建 datahub 消费者用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "user_datahub_consumer" --password "$consumer_password" --tags ""
mq_declare_user "user_datahub_consumer" "$consumer_password" ""
info "创建用户: user_datahub_consumer"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare permissions --vhost "$VHOST" --user "user_datahub_consumer" \
--configure "^(main\\.exchange|errors\\.exchange|dlx\\..*)|(.*\\.queue)$" \
--write "^(orders|products|refunds|inventory).*\\.queue$|(dlx\\..*)|(errors\\.exchange)|(.*\\.errors\\.exchange)$" \
--read "^(main\\.exchange|(orders|products|refunds|inventory).*\\.queue|dlx\\..*)$"
mq_set_permissions "user_datahub_consumer" \
"^(main\\.exchange|errors\\.exchange|dlx\\..*)|(.*\\.queue)$" \
"^(orders|products|refunds|inventory).*\\.queue$|(dlx\\..*)|(errors\\.exchange)|(.*\\.errors\\.exchange)$" \
"^(main\\.exchange|(orders|products|refunds|inventory).*\\.queue|dlx\\..*)$"
info "配置 consumer 用户权限"
# 创建运维监控用户
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "user_ops" --password "$ops_password" --tags ""
mq_declare_user "user_ops" "$ops_password" ""
info "创建用户: user_ops"
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare permissions --vhost "$VHOST" --user "user_ops" \
--configure "^errors\\..*$" --write "" --read "^errors\\.queue$"
mq_set_permissions "user_ops" \
"^errors\\..*$" "" "^errors\\.queue$"
info "配置 ops 用户权限"
}
@@ -600,25 +718,41 @@ cleanup_existing() {
# 检查 VHost 是否存在
echo ""
echo "检查 VHost: $VHOST 是否存在..."
VHOST_EXISTS=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list vhosts | grep -w "$VHOST" || true)
local vhost_list
vhost_list=$(mq_list_vhosts)
local vhost_exists
vhost_exists=$(echo "$vhost_list" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
foreach ($data ?? [] as $v) {
if ($v["name"] === "'"$VHOST"'") { echo "yes"; exit; }
}
' 2>/dev/null)
if [[ -n "$VHOST_EXISTS" ]]; then
if [[ "$vhost_exists" == "yes" ]]; then
warn "VHost '$VHOST' 已存在,将删除所有现有配置..."
# 获取所有平台用户并删除
local users=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list users name -f tsv 2>/dev/null | grep "^user_" || true)
local users_json
users_json=$(mq_list_users)
local users
users=$(echo "$users_json" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
foreach ($data ?? [] as $u) {
if (strpos($u["name"], "user_") === 0) {
echo $u["name"] . "\n";
}
}
' 2>/dev/null)
for user in $users; do
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete user --name "$user" 2>/dev/null || true
while IFS= read -r user; do
if [[ -n "$user" ]]; then
mq_delete_user "$user"
info "删除用户: $user"
done
fi
done <<< "$users"
# 删除 VHost
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
delete vhost --name "$VHOST"
mq_delete_vhost "$VHOST"
info "VHost '$VHOST' 及其所有资源已删除"
else
echo "VHost '$VHOST' 不存在"
@@ -704,7 +838,9 @@ cmd_init() {
echo "- ${#PLATFORMS[@]} 个平台配置"
echo "- $((${#PLATFORMS[@]} + 2)) 个用户"
echo ""
echo "密码配置: $ENV_FILE (MQ_PASSWORD_* 变量)"
echo "密码配置: $ENV_FILE"
echo " (宿主机路径: /var/container/share/rabbitmq_passwords.env)"
echo " 请手动将密码合并到宿主机 /opt/datahub/backend/.env"
echo "用户配置: $MQ_USER_CONFIG (动态从数据库读取)"
}
@@ -723,18 +859,32 @@ cmd_add() {
echo "========================================${NC}"
# 检查 VHost 是否存在
VHOST_EXISTS=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list vhosts | grep -w "$VHOST" || true)
local vhost_list
vhost_list=$(mq_list_vhosts)
local vhost_exists
vhost_exists=$(echo "$vhost_list" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
foreach ($data ?? [] as $v) {
if ($v["name"] === "'"$VHOST"'") { echo "yes"; exit; }
}
' 2>/dev/null)
if [[ -z "$VHOST_EXISTS" ]]; then
if [[ "$vhost_exists" != "yes" ]]; then
error "VHost '$VHOST' 不存在,请先运行 '$0 init' 初始化"
fi
# 检查平台是否已存在
local exchange_exists=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list exchanges --vhost "$VHOST" | grep -w "${platform}.exchange" || true)
local exchanges_json
exchanges_json=$(mq_list_exchanges)
local exchange_exists
exchange_exists=$(echo "$exchanges_json" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
foreach ($data ?? [] as $e) {
if ($e["name"] === "'"${platform}.exchange"'") { echo "yes"; exit; }
}
' 2>/dev/null)
if [[ -n "$exchange_exists" ]]; then
if [[ "$exchange_exists" == "yes" ]]; then
error "平台 '$platform' 已存在"
fi
@@ -833,17 +983,23 @@ cmd_reset_password() {
esac
# 检查用户是否存在
local user_exists=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list users 2>/dev/null | grep -w "${mq_user}" || true)
local users_json
users_json=$(mq_list_users)
local user_exists
user_exists=$(echo "$users_json" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
foreach ($data ?? [] as $u) {
if ($u["name"] === "'"${mq_user}"'") { echo "yes"; exit; }
}
' 2>/dev/null)
if [[ -z "$user_exists" ]]; then
if [[ "$user_exists" != "yes" ]]; then
error "用户 '$mq_user' 不存在"
fi
# 更新 RabbitMQ 用户密码
echo ""
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
declare user --name "$mq_user" --password "$new_password" --tags ""
mq_declare_user "$mq_user" "$new_password" ""
info "RabbitMQ 用户密码已更新: $mq_user"
# 更新 .env 文件
@@ -868,10 +1024,17 @@ cmd_list() {
echo "========================================${NC}"
# 检查 VHost 是否存在
VHOST_EXISTS=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list vhosts 2>/dev/null | grep -w "$VHOST" || true)
local vhost_list
vhost_list=$(mq_list_vhosts)
local vhost_exists
vhost_exists=$(echo "$vhost_list" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
foreach ($data ?? [] as $v) {
if ($v["name"] === "'"$VHOST"'") { echo "yes"; exit; }
}
' 2>/dev/null)
if [[ -z "$VHOST_EXISTS" ]]; then
if [[ "$vhost_exists" != "yes" ]]; then
warn "VHost '$VHOST' 不存在"
return
fi
@@ -879,11 +1042,26 @@ cmd_list() {
# 获取所有平台 Exchange (排除系统 Exchange)
echo ""
echo "MQ 中的平台:"
local exchanges=$(rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
list exchanges --vhost "$VHOST" name -f tsv 2>/dev/null | grep -E "^[a-z0-9_]+\.exchange$" | grep -v "^main\." | grep -v "^errors\." | sed 's/\.exchange$//' | sort)
local exchanges_json
exchanges_json=$(mq_list_exchanges)
local platforms
platforms=$(echo "$exchanges_json" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
foreach ($data ?? [] as $e) {
$name = $e["name"];
// 匹配 xxx.exchange 但排除 main.exchange, errors.exchange, xxx.errors.exchange, dlx.xxx
if (preg_match("/^([a-z0-9_]+)\.exchange$/", $name, $m)
&& $m[1] !== "main"
&& $m[1] !== "errors"
&& !str_contains($name, ".errors.exchange")
&& !str_starts_with($name, "dlx.")) {
echo $m[1] . "\n";
}
}
' 2>/dev/null | sort)
if [[ -n "$exchanges" ]]; then
echo "$exchanges" | while read platform; do
if [[ -n "$platforms" ]]; then
echo "$platforms" | while read platform; do
echo " - $platform"
done
else
@@ -910,8 +1088,20 @@ cmd_version() {
echo -e "========================================${NC}"
echo ""
rabbitmqadmin -H $RABBITMQ_HOST -P $RABBITMQ_PORT -u $RABBITMQ_USER -p $RABBITMQ_PASS \
show overview 2>/dev/null || error "无法连接到 RabbitMQ 服务器"
local overview
overview=$(mq_api_call GET "/overview" 2>/dev/null) || error "无法连接到 RabbitMQ 服务器"
echo "$overview" | php -r '
$data = json_decode(file_get_contents("php://stdin"), true);
if ($data) {
echo "RabbitMQ 版本: " . ($data["rabbitmq_version"] ?? "未知") . "\n";
echo "Erlang 版本: " . ($data["erlang_version"] ?? "未知") . "\n";
echo "Management 版本: " . ($data["management_version"] ?? "未知") . "\n";
echo "集群名称: " . ($data["cluster_name"] ?? "未知") . "\n";
} else {
echo "无法解析服务器信息\n";
}
' 2>/dev/null || error "无法解析服务器信息"
}
# ============================================================================
+2
View File
@@ -20,6 +20,8 @@
"hyperf/command": "~3.1.0",
"hyperf/config": "~3.1.0",
"hyperf/constants": "~3.1.0",
"hyperf/crontab": "~3.1.0",
"gokure/hyperf-cors": "^2.1",
"hyperf/database-pgsql": "^3.1",
"hyperf/db-connection": "~3.1.0",
"hyperf/engine": "^2.10",
+10920
View File
File diff suppressed because it is too large Load Diff
+38
View File
@@ -0,0 +1,38 @@
<?php
declare(strict_types=1);
/**
* CORS 配置
*
* 前端 Vue (http://server:8080) 跨源访问后端 (http://server:9501) 时必需。
* 由于前端走 Bearer tokenAuthorization header)而非 cookie
* supports_credentials 保持 falseallowed_origins 可使用 '*'。
*
* 生产环境如要收紧,把 allowed_origins 改成具体域名列表。
*/
return [
// 仅对 /api/* 启用 CORS(前端调用都带此前缀)
'paths' => ['api/*'],
// 允许的方法
'allowed_methods' => ['*'],
// 允许的源(通配;若改成 supports_credentials=true,必须列具体域名)
'allowed_origins' => ['*'],
// 允许的源(正则匹配,按需启用)
'allowed_origins_patterns' => [],
// 允许的请求头
'allowed_headers' => ['*'],
// 暴露给前端的响应头
'exposed_headers' => [],
// 预检请求缓存时间(秒)
'max_age' => 7200,
// 是否允许携带凭证(cookie / 客户端证书);本项目用 Bearer token,保持 false
'supports_credentials' => false,
];
+19
View File
@@ -0,0 +1,19 @@
<?php
declare(strict_types=1);
use App\Service\OrderAggregatesRefreshJob;
use Hyperf\Crontab\Crontab;
return [
'enable' => true,
'crontab' => [
// 每天 02:07 处理 aggregate_refresh_queue 中的滞后聚合刷新。
// 02 时段为最低流量段;分钟取 :07 避开整点全互联网定时任务集中触发。
(new Crontab())
->setName('OrderAggregatesRefresh')
->setRule('7 2 * * *')
->setCallback([OrderAggregatesRefreshJob::class, 'run'])
->setMemo('每天 02:07 处理 aggregate_refresh_queue 中的滞后聚合刷新'),
],
];
+1
View File
@@ -11,4 +11,5 @@ declare(strict_types=1);
*/
return [
// 可以在这里配置接口到实现的绑定
App\Service\AggregateRefresherInterface::class => App\Service\ContinuousAggregateRefresher::class,
];
+1
View File
@@ -11,6 +11,7 @@ declare(strict_types=1);
*/
return [
'http' => [
\Gokure\HyperfCors\CorsMiddleware::class,
\App\Middleware\RequestLogMiddleware::class,
],
];
+1
View File
@@ -10,4 +10,5 @@ declare(strict_types=1);
* @license https://github.com/hyperf/hyperf/blob/master/LICENSE
*/
return [
Hyperf\Crontab\Process\CrontabDispatcherProcess::class,
];
+41
View File
@@ -0,0 +1,41 @@
#!/bin/sh
# Datahub backend container entrypoint.
# Flow: wait_tcp pg/mq -> migrate --force -> app:install -> exec swoole.
set -eu
wait_tcp() {
host="$1"
port="$2"
label="$3"
timeout="${4:-90}"
echo "[entrypoint] waiting for ${label} @ ${host}:${port} (up to ${timeout}s)"
i=0
while [ "$i" -lt "$timeout" ]; do
if php -r "exit(@fsockopen('${host}', ${port}, \$e, \$s, 1) ? 0 : 1);" 2>/dev/null; then
echo "[entrypoint] ${label} reachable"
return 0
fi
i=$((i + 1))
sleep 1
done
echo "[entrypoint] ${label} @ ${host}:${port} unreachable after ${timeout}s" >&2
return 1
}
DB_HOST="${DB_HOST:-host.containers.internal}"
DB_PORT="${DB_PORT:-5432}"
AMQP_HOST="${AMQP_HOST:-host.containers.internal}"
AMQP_PORT="${AMQP_PORT:-5672}"
WAIT_TIMEOUT="${ENTRYPOINT_WAIT_TIMEOUT:-90}"
wait_tcp "${DB_HOST}" "${DB_PORT}" "postgres" "${WAIT_TIMEOUT}"
wait_tcp "${AMQP_HOST}" "${AMQP_PORT}" "rabbitmq" "${WAIT_TIMEOUT}"
echo "[entrypoint] running migrate --force"
php /var/www/bin/hyperf.php migrate --force
echo "[entrypoint] running app:install"
php /var/www/bin/hyperf.php app:install
echo "[entrypoint] handing off to: $*"
exec "$@"
@@ -0,0 +1,20 @@
<?php
declare(strict_types=1);
use Hyperf\DbConnection\Db;
use Hyperf\Database\Migrations\Migration;
return new class extends Migration
{
public function up(): void
{
Db::statement('CREATE EXTENSION IF NOT EXISTS timescaledb');
}
public function down(): void
{
// 不主动 DROP EXTENSIONexisting hypertables 依赖该扩展,drop 会破坏数据。
// 完全清空数据库时手动执行 `DROP EXTENSION timescaledb CASCADE`。
}
};
@@ -0,0 +1,52 @@
<?php
declare(strict_types=1);
use Hyperf\DbConnection\Db;
use Hyperf\Database\Migrations\Migration;
return new class extends Migration
{
public function up(): void
{
// 连续聚合视图:按订单创建日期日聚合(含未付订单)。
// WITH NO DATA:视图创建时不立即物化,由 P22.3 的回填命令一次性填充历史数据。
Db::statement(<<<'SQL'
CREATE MATERIALIZED VIEW orders_daily_by_created
WITH (timescaledb.continuous) AS
SELECT
time_bucket('1 day', created_date) AS day,
company_id,
platform_id,
store_id,
COUNT(*) AS total_orders,
COUNT(*) FILTER (WHERE paid_date IS NOT NULL) AS paid_orders,
COUNT(*) FILTER (WHERE paid_date IS NULL) AS unpaid_orders,
SUM(total_amount) AS sum_total_amount,
SUM(total_paid) AS sum_total_paid,
SUM(total_received) AS sum_total_received,
AVG(total_amount) AS avg_total_amount,
AVG(total_paid) FILTER (WHERE paid_date IS NOT NULL) AS avg_paid_amount,
SUM(freight_fee) AS sum_freight_fee,
SUM(tax_fee) AS sum_tax_fee,
SUM(commission_fee) AS sum_commission_fee,
SUM(discount_fee) AS sum_discount_fee
FROM orders
GROUP BY day, company_id, platform_id, store_id
WITH NO DATA;
SQL);
// 复合索引:覆盖"最近 N 日按 X 维度"与"指定 X 全历史时间序列"两类高频查询。
Db::statement('CREATE INDEX IF NOT EXISTS idx_orders_daily_by_created_day_company ON orders_daily_by_created (day DESC, company_id)');
Db::statement('CREATE INDEX IF NOT EXISTS idx_orders_daily_by_created_day_platform ON orders_daily_by_created (day DESC, platform_id)');
Db::statement('CREATE INDEX IF NOT EXISTS idx_orders_daily_by_created_day_store ON orders_daily_by_created (day DESC, store_id)');
Db::statement('CREATE INDEX IF NOT EXISTS idx_orders_daily_by_created_company_day ON orders_daily_by_created (company_id, day DESC)');
Db::statement('CREATE INDEX IF NOT EXISTS idx_orders_daily_by_created_store_day ON orders_daily_by_created (store_id, day DESC)');
}
public function down(): void
{
// CASCADE 一并 drop 索引以及后续阶段附加的刷新策略。
Db::statement('DROP MATERIALIZED VIEW IF EXISTS orders_daily_by_created CASCADE');
}
};
@@ -0,0 +1,69 @@
<?php
declare(strict_types=1);
use Hyperf\DbConnection\Db;
use Hyperf\Database\Migrations\Migration;
return new class extends Migration
{
public function up(): void
{
// PG 原生物化视图:按付款日聚合(仅已付订单)。
// 不带 WITH (timescaledb.continuous)TimescaleDB 连续聚合的 time_bucket
// 必须基于 hypertable 主时间列(created_date),无法对 paid_date 分桶。
// WITH NO DATA:避免 migrate 时长时间锁;首次填充由 P22.3 回填命令执行。
Db::statement(<<<'SQL'
CREATE MATERIALIZED VIEW orders_daily_by_paid AS
SELECT
paid_date::date AS day,
company_id,
platform_id,
store_id,
COUNT(*) AS paid_orders,
SUM(total_amount) AS sum_total_amount,
SUM(total_paid) AS sum_total_paid,
SUM(total_received) AS sum_total_received,
AVG(total_paid) AS avg_paid_amount,
SUM(freight_fee) AS sum_freight_fee,
SUM(tax_fee) AS sum_tax_fee,
SUM(commission_fee) AS sum_commission_fee,
SUM(discount_fee) AS sum_discount_fee
FROM orders
WHERE paid_date IS NOT NULL
GROUP BY day, company_id, platform_id, store_id
WITH NO DATA;
SQL);
// UNIQUE 索引:REFRESH MATERIALIZED VIEW CONCURRENTLY 的前置条件,缺失会直接报错。
Db::statement('CREATE UNIQUE INDEX idx_orders_daily_by_paid_unique ON orders_daily_by_paid (day, company_id, platform_id, store_id)');
// 5 复合索引:覆盖"最近 N 日按 X 维度"与"指定 X 全历史时间序列"两类高频查询。
Db::statement('CREATE INDEX idx_orders_daily_by_paid_day_company ON orders_daily_by_paid (day DESC, company_id)');
Db::statement('CREATE INDEX idx_orders_daily_by_paid_day_platform ON orders_daily_by_paid (day DESC, platform_id)');
Db::statement('CREATE INDEX idx_orders_daily_by_paid_day_store ON orders_daily_by_paid (day DESC, store_id)');
Db::statement('CREATE INDEX idx_orders_daily_by_paid_company_day ON orders_daily_by_paid (company_id, day DESC)');
Db::statement('CREATE INDEX idx_orders_daily_by_paid_store_day ON orders_daily_by_paid (store_id, day DESC)');
// by_created 连续聚合自动刷新策略:每小时刷新近 3 天水位线,1 小时写入缓冲避免与热数据冲突。
// 3 天前的修改由 Stage 23 aggregate_refresh_queue 兜底。
Db::statement(<<<'SQL'
SELECT add_continuous_aggregate_policy(
'orders_daily_by_created',
start_offset => INTERVAL '3 days',
end_offset => INTERVAL '1 hour',
schedule_interval => INTERVAL '1 hour'
);
SQL);
}
public function down(): void
{
// 先 remove policy,再 drop 视图 CASCADE。
// if_exists => true 处理 by_created 视图已被 P22.1 down CASCADE 间接带走的情况。
Db::statement("SELECT remove_continuous_aggregate_policy('orders_daily_by_created', if_exists => true)");
// CASCADE 一并 drop UNIQUE 索引和 5 复合索引。
Db::statement('DROP MATERIALIZED VIEW IF EXISTS orders_daily_by_paid CASCADE');
}
};
@@ -0,0 +1,32 @@
<?php
declare(strict_types=1);
use Hyperf\Database\Schema\Schema;
use Hyperf\Database\Schema\Blueprint;
use Hyperf\Database\Migrations\Migration;
use Hyperf\DbConnection\Db;
return new class extends Migration
{
public function up(): void
{
Schema::create('aggregate_refresh_queue', function (Blueprint $table) {
$table->date('refresh_date')->comment('待刷新的聚合日');
$table->string('aggregate_view', 100)->comment('视图名(orders_daily_by_created / orders_daily_by_paid');
$table->timestampTz('created_at')->useCurrent()->comment('入队时间');
$table->primary(['refresh_date', 'aggregate_view'], 'pk_aggregate_refresh_queue');
$table->index('created_at', 'idx_aggregate_refresh_queue_created_at');
});
// Hyperf PostgresGrammar::compilePrimary 丢弃自定义 index 名(vendor/hyperf/database-pgsql/.../PostgresGrammar.php:230),
// PG 默认用 {table}_pkey;这里显式重命名以满足计划验收要求(名称 pk_aggregate_refresh_queue
Db::statement('ALTER TABLE aggregate_refresh_queue RENAME CONSTRAINT aggregate_refresh_queue_pkey TO pk_aggregate_refresh_queue');
}
public function down(): void
{
Schema::dropIfExists('aggregate_refresh_queue');
}
};
@@ -0,0 +1,161 @@
<?php
declare(strict_types=1);
namespace HyperfTest\Cases\Integration\Materialization;
use App\Model\AggregateRefreshQueue;
use App\Model\Role;
use App\Model\User;
use Carbon\Carbon;
use HyperfTest\TestCase;
use Qbhy\HyperfAuth\AuthManager;
use function Hyperf\Support\make;
/**
* AdminMaterializationController 集成测试
*
* @internal
* @coversNothing
*/
class AdminMaterializationControllerTest extends TestCase
{
protected function fetchAdminRole(): Role
{
return Role::query()->where('name', 'administrator')->firstOrFail();
}
protected function getAdminAuthToken(): string
{
$admin_role = $this->fetchAdminRole();
$user = User::query()
->where('status', 1)
->where('role_id', $admin_role->id)
->first();
if (!$user) {
$this->markTestSkipped('没有可用的 administrator 用户,无法测试');
}
$auth = make(AuthManager::class);
return $auth->guard('jwt')->login($user);
}
protected function adminHeaders(): array
{
return ['Authorization' => 'Bearer ' . $this->getAdminAuthToken()];
}
protected function getNonAdminToken(): array
{
$suffix = 'mat_nonadmin_' . uniqid();
$user = User::query()->create([
'username' => $suffix,
'password' => 'Pass_' . $suffix,
'email' => $suffix . '@example.com',
'status' => 1,
'api_key_enabled' => true,
]);
$auth = make(AuthManager::class);
$token = $auth->guard('jwt')->login($user);
return ['Authorization' => 'Bearer ' . $token];
}
public function test_queue_lists_pending(): void
{
$date = '2030-12-31';
$view = 'orders_daily_by_created';
AggregateRefreshQueue::query()->insertOrIgnore([[
'refresh_date' => $date,
'aggregate_view' => $view,
'created_at' => Carbon::now(),
]]);
try {
$response = $this->get(
'/api/v1/admin/materialization/queue',
['view' => $view, 'from' => $date, 'to' => $date],
$this->adminHeaders()
);
$response->assertStatus(200);
$response->assertJsonPath('code', 0);
$body = json_decode($response->getBody()->getContents(), true);
$this->assertArrayHasKey('items', $body['data']);
$this->assertArrayHasKey('total', $body['data']);
$this->assertArrayHasKey('page', $body['data']);
$this->assertArrayHasKey('per_page', $body['data']);
$this->assertGreaterThanOrEqual(1, $body['data']['total']);
$found = false;
foreach ($body['data']['items'] as $item) {
if ($item['refresh_date'] === $date && $item['aggregate_view'] === $view) {
$found = true;
break;
}
}
$this->assertTrue($found, '应在 queue 列表中找到刚插入的 fixture 行');
} finally {
AggregateRefreshQueue::query()
->where('refresh_date', $date)
->where('aggregate_view', $view)
->delete();
}
}
public function test_refresh_validates_view_whitelist(): void
{
$response = $this->post(
'/api/v1/admin/materialization/refresh',
[
'view' => 'evil_view',
'from' => '2026-01-01 00:00:00+00',
'to' => '2026-01-02 00:00:00+00',
],
$this->adminHeaders()
);
$response->assertStatus(400);
$body = json_decode($response->getBody()->getContents(), true);
$this->assertSame(400, $body['code']);
$this->assertStringContainsString('view', $body['message']);
}
public function test_aggregates_returns_lag_seconds(): void
{
$response = $this->get('/api/v1/admin/materialization/aggregates', [], $this->adminHeaders());
$response->assertStatus(200);
$response->assertJsonPath('code', 0);
$body = json_decode($response->getBody()->getContents(), true);
$this->assertArrayHasKey('items', $body['data']);
$items = $body['data']['items'];
$this->assertNotEmpty($items, 'aggregates 应至少返回一条连续聚合记录(orders_daily_by_created');
$this->assertArrayHasKey('view_name', $items[0]);
$this->assertArrayHasKey('lag_seconds', $items[0]);
}
public function test_jobs_lists_refresh_policy(): void
{
$response = $this->get('/api/v1/admin/materialization/jobs', [], $this->adminHeaders());
$response->assertStatus(200);
$response->assertJsonPath('code', 0);
$body = json_decode($response->getBody()->getContents(), true);
$this->assertArrayHasKey('items', $body['data']);
// by_created 注册了 1 条 policy_refresh_continuous_aggregateby_paid 由 Hyperf Crontab 调度,不入此表
$this->assertGreaterThanOrEqual(1, count($body['data']['items']));
$this->assertSame('policy_refresh_continuous_aggregate', $body['data']['items'][0]['proc_name']);
}
public function test_non_admin_blocked(): void
{
$headers = $this->getNonAdminToken();
$response = $this->get('/api/v1/admin/materialization/queue', [], $headers);
$response->assertStatus(403);
}
}
@@ -0,0 +1,94 @@
<?php
declare(strict_types=1);
namespace HyperfTest\Cases\Integration\Materialization;
use Hyperf\DbConnection\Db;
use PHPUnit\Framework\TestCase;
/**
* P22.3 物化对象集成测试:跨两类物化对象 + 跨命名空间索引的存在性断言。
*
* 与 P22.2 烟雾测试 `System/MaterializedViewSmokeTest` 各自关注点不同:
* - 烟雾测试:单点测试,专测 by_paid 视图与索引
* - 本测试:跨两类对象(连续聚合 + PG 物化视图)+ 跨命名空间索引(pg_indexes 不区分视图类型)
*
* @internal
* @coversNothing
*/
class MaterializationObjectsTest extends TestCase
{
/**
* @template T
* @param callable(): T $callback
* @return T
*/
protected function runInCoroutine(callable $callback): mixed
{
if (\Swoole\Coroutine::getCid() > 0) {
return $callback();
}
$result = null;
$exception = null;
\Swoole\Coroutine\run(static function () use ($callback, &$result, &$exception): void {
try {
$result = $callback();
} catch (\Throwable $e) {
$exception = $e;
}
});
if ($exception !== null) {
throw $exception;
}
return $result;
}
public function test_continuous_aggregate_exists(): void
{
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT view_name FROM timescaledb_information.continuous_aggregates
WHERE view_name = 'orders_daily_by_created'"
));
$this->assertCount(1, $rows);
}
public function test_pg_materialized_view_exists(): void
{
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT matviewname FROM pg_matviews
WHERE matviewname = 'orders_daily_by_paid'"
));
$this->assertCount(1, $rows);
}
public function test_refresh_policy_registered(): void
{
// 显式过滤 hypertable_name,避免未来追加其他连续聚合策略时此断言无关地失败。
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT hypertable_name FROM timescaledb_information.jobs
WHERE proc_name = 'policy_refresh_continuous_aggregate'
AND hypertable_name = 'orders_daily_by_created'"
));
$this->assertCount(1, $rows);
}
public function test_by_created_indexes_present(): void
{
// P22.1 创建 5 复合索引;chunk 索引名是 `_hyper_*` 不会被 LIKE 'idx_*' 匹配。
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT indexname FROM pg_indexes WHERE indexname LIKE 'idx_orders_daily_by_created%'"
));
$this->assertCount(5, $rows);
}
public function test_by_paid_indexes_present(): void
{
// P22.2 创建 1 UNIQUE + 5 复合 = 6 索引。
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT indexname FROM pg_indexes WHERE indexname LIKE 'idx_orders_daily_by_paid%'"
));
$this->assertCount(6, $rows);
}
}
@@ -0,0 +1,69 @@
<?php
declare(strict_types=1);
namespace HyperfTest\Cases\Integration\System;
use Hyperf\DbConnection\Db;
use PHPUnit\Framework\TestCase;
/**
* P22.2 物化层 schema 烟雾测试:断言迁移已应用、索引齐备、刷新策略已注册。
*
* @internal
* @coversNothing
*/
class MaterializedViewSmokeTest extends TestCase
{
/**
* @template T
* @param callable(): T $callback
* @return T
*/
protected function runInCoroutine(callable $callback): mixed
{
if (\Swoole\Coroutine::getCid() > 0) {
return $callback();
}
$result = null;
$exception = null;
\Swoole\Coroutine\run(static function () use ($callback, &$result, &$exception): void {
try {
$result = $callback();
} catch (\Throwable $e) {
$exception = $e;
}
});
if ($exception !== null) {
throw $exception;
}
return $result;
}
public function test_orders_daily_by_paid_matview_exists(): void
{
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT matviewname FROM pg_matviews WHERE matviewname = 'orders_daily_by_paid'"
));
$this->assertCount(1, $rows);
}
public function test_orders_daily_by_paid_has_six_indexes(): void
{
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT indexname FROM pg_indexes WHERE tablename = 'orders_daily_by_paid'"
));
$this->assertCount(6, $rows, '应有 1 UNIQUE + 5 复合 = 6 个索引');
}
public function test_orders_daily_by_created_refresh_policy_registered(): void
{
$rows = $this->runInCoroutine(static fn () => Db::select(
"SELECT job_id FROM timescaledb_information.jobs
WHERE proc_name = 'policy_refresh_continuous_aggregate'
AND hypertable_name = 'orders_daily_by_created'"
));
$this->assertCount(1, $rows);
}
}
@@ -0,0 +1,89 @@
<?php
declare(strict_types=1);
namespace HyperfTest\Cases\Unit\Model;
use App\Model\AggregateRefreshQueue;
use Carbon\Carbon;
use PHPUnit\Framework\TestCase;
/**
* @internal
* @coversNothing
*/
class AggregateRefreshQueueTest extends TestCase
{
protected function runInCoroutine(callable $callback): void
{
if (\Swoole\Coroutine::getCid() > 0) {
$callback();
return;
}
$exception = null;
\Swoole\Coroutine\run(static function () use ($callback, &$exception): void {
try {
$callback();
} catch (\Throwable $e) {
$exception = $e;
}
});
if ($exception) {
throw $exception;
}
}
private function uniqueRefreshDate(): string
{
// 与 ApiKeyTest 的 bin2hex(random_bytes(4)) 同源策略:用唯一性而非全表清理来隔离用例
return sprintf('20%02d-%02d-%02d', random_int(50, 99), random_int(1, 12), random_int(1, 28));
}
public function test_attributes_persist_correctly(): void
{
$this->runInCoroutine(function (): void {
$unique_date = $this->uniqueRefreshDate();
try {
AggregateRefreshQueue::query()->create([
'refresh_date' => $unique_date,
'aggregate_view' => 'orders_daily_by_created',
'created_at' => Carbon::now(),
]);
$row = AggregateRefreshQueue::query()->where('refresh_date', $unique_date)->first();
$this->assertNotNull($row);
$this->assertSame($unique_date, $row->refresh_date);
$this->assertSame('orders_daily_by_created', $row->aggregate_view);
$this->assertInstanceOf(Carbon::class, $row->created_at);
} finally {
AggregateRefreshQueue::query()->where('refresh_date', $unique_date)->delete();
}
});
}
public function test_insert_or_ignore_dedups_on_composite_pk(): void
{
$this->runInCoroutine(function (): void {
$unique_date = $this->uniqueRefreshDate();
$payload = [
'refresh_date' => $unique_date,
'aggregate_view' => 'orders_daily_by_created',
'created_at' => Carbon::now(),
];
try {
AggregateRefreshQueue::query()->insertOrIgnore($payload);
AggregateRefreshQueue::query()->insertOrIgnore($payload);
$count = AggregateRefreshQueue::query()->where('refresh_date', $unique_date)->count();
$this->assertSame(1, $count);
} finally {
AggregateRefreshQueue::query()->where('refresh_date', $unique_date)->delete();
}
});
}
}
@@ -0,0 +1,164 @@
<?php
declare(strict_types=1);
namespace HyperfTest\Cases\Unit\Platform;
use App\Model\AggregateRefreshQueue;
use App\Platform\OrderConsumer;
use Carbon\Carbon;
use PHPUnit\Framework\TestCase;
use ReflectionMethod;
/**
* @internal
* @coversNothing
*/
class OrderConsumerEnqueueTest extends TestCase
{
protected function runInCoroutine(callable $callback): void
{
if (\Swoole\Coroutine::getCid() > 0) {
$callback();
return;
}
$exception = null;
\Swoole\Coroutine\run(static function () use ($callback, &$exception): void {
try {
$callback();
} catch (\Throwable $e) {
$exception = $e;
}
});
if ($exception) {
throw $exception;
}
}
/**
* 调用 protected enqueueAffectedDates 方法
*/
protected function invokeEnqueue(array $payloads): void
{
$consumer = new OrderConsumer();
$method = new ReflectionMethod($consumer, 'enqueueAffectedDates');
$method->setAccessible(true);
$method->invoke($consumer, $payloads);
}
/**
* 清理指定日期的队列条目(前置 + 后置双保险)
*/
protected function cleanupQueue(string $date): void
{
AggregateRefreshQueue::query()
->where('refresh_date', $date)
->where('aggregate_view', 'orders_daily_by_created')
->delete();
}
/**
* 当日订单不应入队(在自动刷新策略覆盖窗口内)
*/
public function test_today_payload_does_not_enqueue(): void
{
$this->runInCoroutine(function (): void {
$today = Carbon::now()->toDateString();
$payloads = [['created_date' => Carbon::now()->format('Y-m-d H:i:sP')]];
$this->cleanupQueue($today);
try {
$this->invokeEnqueue($payloads);
$row = AggregateRefreshQueue::query()
->where('refresh_date', $today)
->where('aggregate_view', 'orders_daily_by_created')
->first();
$this->assertNull($row);
} finally {
$this->cleanupQueue($today);
}
});
}
/**
* 2 天前订单不应入队(仍在自动刷新策略 [now-3d, now-1h] 窗口内)
*/
public function test_two_days_ago_payload_does_not_enqueue(): void
{
$this->runInCoroutine(function (): void {
$two_days_ago = Carbon::now()->subDays(2)->toDateString();
$payloads = [['created_date' => Carbon::now()->subDays(2)->format('Y-m-d H:i:sP')]];
$this->cleanupQueue($two_days_ago);
try {
$this->invokeEnqueue($payloads);
$row = AggregateRefreshQueue::query()
->where('refresh_date', $two_days_ago)
->where('aggregate_view', 'orders_daily_by_created')
->first();
$this->assertNull($row);
} finally {
$this->cleanupQueue($two_days_ago);
}
});
}
/**
* 30 天前订单应入队 by_created(自动策略未覆盖,需补刷)
*/
public function test_old_payload_enqueues_by_created(): void
{
$this->runInCoroutine(function (): void {
$old_date = Carbon::now()->subDays(30)->toDateString();
$payloads = [['created_date' => Carbon::now()->subDays(30)->format('Y-m-d H:i:sP')]];
$this->cleanupQueue($old_date);
try {
$this->invokeEnqueue($payloads);
$row = AggregateRefreshQueue::query()
->where('refresh_date', $old_date)
->where('aggregate_view', 'orders_daily_by_created')
->first();
$this->assertNotNull($row);
$this->assertSame($old_date, $row->refresh_date);
$this->assertSame('orders_daily_by_created', $row->aggregate_view);
} finally {
$this->cleanupQueue($old_date);
}
});
}
/**
* 同日期重复入队由复合主键 + insertOrIgnore 防重
*/
public function test_dedup_via_insert_or_ignore(): void
{
$this->runInCoroutine(function (): void {
$old_date = Carbon::now()->subDays(45)->toDateString();
$payloads = [
['created_date' => Carbon::now()->subDays(45)->format('Y-m-d H:i:sP')],
['created_date' => Carbon::now()->subDays(45)->format('Y-m-d H:i:sP')],
];
$this->cleanupQueue($old_date);
try {
// 单批次内同日期 → $unique_dates map 去重
$this->invokeEnqueue($payloads);
// 跨批次同日期 → 复合主键 insertOrIgnore 防重
$this->invokeEnqueue($payloads);
$count = AggregateRefreshQueue::query()
->where('refresh_date', $old_date)
->where('aggregate_view', 'orders_daily_by_created')
->count();
$this->assertSame(1, $count);
} finally {
$this->cleanupQueue($old_date);
}
});
}
}
@@ -0,0 +1,178 @@
<?php
declare(strict_types=1);
namespace HyperfTest\Cases\Unit\Service;
use App\Model\AggregateRefreshQueue;
use App\Service\AggregateRefresherInterface;
use App\Service\OrderAggregatesRefreshJob;
use Carbon\Carbon;
use PHPUnit\Framework\TestCase;
use RuntimeException;
/**
* 测试用 stub Refresher,记录每次 refresh 调用
*/
class RecordingRefresher implements AggregateRefresherInterface
{
/** @var array<int, array{view: string, date: string}> */
public array $calls = [];
public function refresh(string $view, string $refresh_date): void
{
$this->calls[] = ['view' => $view, 'date' => $refresh_date];
}
}
/**
* 测试用 stub Refresher,每次调用抛异常(模拟刷新失败)
*/
class ThrowingRefresher implements AggregateRefresherInterface
{
public function refresh(string $view, string $refresh_date): void
{
throw new RuntimeException('refresh failed');
}
}
/**
* @internal
* @coversNothing
*/
class OrderAggregatesRefreshJobTest extends TestCase
{
protected function runInCoroutine(callable $callback): void
{
if (\Swoole\Coroutine::getCid() > 0) {
$callback();
return;
}
$exception = null;
\Swoole\Coroutine\run(static function () use ($callback, &$exception): void {
try {
$callback();
} catch (\Throwable $e) {
$exception = $e;
}
});
if ($exception) {
throw $exception;
}
}
/**
* 沿用 P23.1 AggregateRefreshQueueTest 的随机未来日期隔离策略,
* 确保不同测试用例间无冲突,也不会污染生产数据。
*/
private function uniqueRefreshDate(): string
{
return sprintf('20%02d-%02d-%02d', random_int(50, 99), random_int(1, 12), random_int(1, 28));
}
/**
* 队列项 created_at < now()-1h 应被处理:调用 refresher 后从队列删除
*/
public function test_processes_old_items_and_clears_queue(): void
{
$this->runInCoroutine(function (): void {
$date_a = $this->uniqueRefreshDate();
$date_b = $this->uniqueRefreshDate();
while ($date_b === $date_a) {
$date_b = $this->uniqueRefreshDate();
}
// 前置清理(防 random 撞历史残留)
AggregateRefreshQueue::query()->whereIn('refresh_date', [$date_a, $date_b])->delete();
try {
$two_hours_ago = Carbon::now()->subHours(2);
AggregateRefreshQueue::query()->insertOrIgnore([
['refresh_date' => $date_a, 'aggregate_view' => 'orders_daily_by_created', 'created_at' => $two_hours_ago],
['refresh_date' => $date_b, 'aggregate_view' => 'orders_daily_by_created', 'created_at' => $two_hours_ago],
]);
$refresher = new RecordingRefresher();
$job = new OrderAggregatesRefreshJob($refresher);
$result = $job->run();
$this->assertSame(2, $result['processed']);
$this->assertSame(0, $result['failed']);
$this->assertCount(2, $refresher->calls);
// 队列已清空(针对本次测试的两条)
$remaining = AggregateRefreshQueue::query()
->whereIn('refresh_date', [$date_a, $date_b])
->count();
$this->assertSame(0, $remaining);
} finally {
AggregateRefreshQueue::query()->whereIn('refresh_date', [$date_a, $date_b])->delete();
}
});
}
/**
* 队列项 created_at >= now()-1h 应被跳过:refresher 无调用,队列保留
*/
public function test_recently_enqueued_items_are_skipped(): void
{
$this->runInCoroutine(function (): void {
$date = $this->uniqueRefreshDate();
AggregateRefreshQueue::query()->where('refresh_date', $date)->delete();
try {
AggregateRefreshQueue::query()->insertOrIgnore([
'refresh_date' => $date,
'aggregate_view' => 'orders_daily_by_created',
'created_at' => Carbon::now()->subMinutes(30), // < 1 hour
]);
$refresher = new RecordingRefresher();
$job = new OrderAggregatesRefreshJob($refresher);
$result = $job->run();
$this->assertSame(0, $result['processed']);
$this->assertSame(0, $result['failed']);
$this->assertCount(0, $refresher->calls);
// 队列项保留
$exists = AggregateRefreshQueue::query()->where('refresh_date', $date)->exists();
$this->assertTrue($exists);
} finally {
AggregateRefreshQueue::query()->where('refresh_date', $date)->delete();
}
});
}
/**
* refresher 抛异常时:队列项保留(下次任务重试),失败计数 +1
*/
public function test_failure_keeps_queue_item(): void
{
$this->runInCoroutine(function (): void {
$date = $this->uniqueRefreshDate();
AggregateRefreshQueue::query()->where('refresh_date', $date)->delete();
try {
AggregateRefreshQueue::query()->insertOrIgnore([
'refresh_date' => $date,
'aggregate_view' => 'orders_daily_by_created',
'created_at' => Carbon::now()->subHours(2),
]);
$job = new OrderAggregatesRefreshJob(new ThrowingRefresher());
$result = $job->run();
$this->assertSame(0, $result['processed']);
$this->assertSame(1, $result['failed']);
// 队列项保留以待重试
$exists = AggregateRefreshQueue::query()->where('refresh_date', $date)->exists();
$this->assertTrue($exists);
} finally {
AggregateRefreshQueue::query()->where('refresh_date', $date)->delete();
}
});
}
}
+43
View File
@@ -0,0 +1,43 @@
# 部署到 /var/container/data/datahub/env/datahub-backend.env
# 仅存放非敏感配置;密码/token 由 podman secret 注入(见 scripts/create-secrets.sh
APP_NAME=datahub
APP_ENV=prod
SCAN_CACHEABLE=true
# --- PostgreSQL (TimescaleDB) ---
DB_DRIVER=pgsql
DB_HOST=datahub-postgres
DB_PORT=5432
DB_DATABASE=datahub
DB_USERNAME=datahub
DB_CHARSET=utf8
DB_COLLATION=utf8_unicode_ci
DB_PREFIX=
DB_SCHEMA=public
DB_TIMEZONE=Asia/Shanghai
DB_SSL_MODE=disable
DB_MAX_IDLE_TIME=60
# DB_PASSWORD 由 podman secret 注入
# --- RabbitMQ ---
AMQP_HOST=datahub-rabbitmq
AMQP_PORT=5672
AMQP_USER=user
AMQP_ADMIN_USER=user
AMQP_VHOST=dataflow
AMQP_MAX_RETRIES=3
AMQP_CONSUMER_DEBUG_DELAY=0
RABBITMQ_MANAGEMENT_PORT=15672
# AMQP_PASSWORD / AMQP_ADMIN_PASSWORD 由 podman secret 注入
# --- JWT ---
JWT_HEADER_NAME=Authorization
SIMPLE_JWT_TTL=7200
SIMPLE_JWT_REFRESH_TTL=2592000
SIMPLE_JWT_PREFIX=dataflow
# SIMPLE_JWT_SECRET 由 podman secret 注入
# --- 外部依赖 ---
TOOLS_HOST=https://store-api-v2.wpic-tools.com/
# TOOLS_TOKEN 由 podman secret 注入
+5
View File
@@ -0,0 +1,5 @@
# 部署到 /var/container/data/datahub/env/datahub-frontend.env
# 浏览器访问后端 API 的完整地址(含协议、主机、端口)
# 由 scripts/configure-env.sh 交互式生成,也可手动维护
API_BASE_URL=http://__SERVER_IP_OR_DOMAIN__:9501
+13
View File
@@ -0,0 +1,13 @@
FROM node:22-alpine AS builder
WORKDIR /app
COPY frontend/package*.json ./
RUN npm ci
COPY frontend/ ./
RUN npm run build
FROM nginx:alpine
COPY --from=builder /app/dist /usr/share/nginx/html
COPY deploy/podman/images/frontend/nginx.conf /etc/nginx/conf.d/default.conf
COPY deploy/podman/images/frontend/docker-entrypoint.sh /docker-entrypoint.d/40-inject-config.sh
RUN chmod +x /docker-entrypoint.d/40-inject-config.sh
EXPOSE 80
@@ -0,0 +1,22 @@
#!/bin/sh
# 由 nginx 官方镜像的入口脚本自动执行(位于 /docker-entrypoint.d/
# 将 config.js 中的占位符替换为运行期环境变量
set -e
CONFIG_FILE=/usr/share/nginx/html/config.js
if [ ! -f "$CONFIG_FILE" ]; then
echo "[entrypoint] $CONFIG_FILE 不存在,跳过注入" >&2
exit 0
fi
if [ -z "${API_BASE_URL:-}" ]; then
echo "[entrypoint] 错误:必须设置 API_BASE_URL 环境变量" >&2
exit 1
fi
# 用 | 作分隔符,避免 URL 里的 / 干扰
sed -i "s|__API_BASE_URL__|${API_BASE_URL}|g" "$CONFIG_FILE"
echo "[entrypoint] 已注入 API_BASE_URL=${API_BASE_URL}"
+21
View File
@@ -0,0 +1,21 @@
server {
listen 80;
server_name _;
root /usr/share/nginx/html;
index index.html;
gzip on;
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml;
gzip_min_length 1024;
location ~* \.(?:css|js|woff2?|ttf|eot|svg|png|jpg|jpeg|gif|ico|webp)$ {
expires 7d;
add_header Cache-Control "public, immutable";
try_files $uri =404;
}
location / {
try_files $uri $uri/ /index.html;
}
}
+11
View File
@@ -0,0 +1,11 @@
FROM docker.io/timescale/timescaledb:latest-pg16
LABEL maintainer="WPIC Datahub" \
app.name="datahub-postgres"
ENV TZ=Asia/Shanghai \
LANG=en_US.utf8
COPY deploy/podman/images/postgres/initdb/ /docker-entrypoint-initdb.d/
EXPOSE 5432
@@ -0,0 +1,11 @@
-- 首次初始化时执行(仅在 $PGDATA 为空时触发)
-- 在 datahub 数据库内启用 TimescaleDB 扩展
-- 迁移 2026_01_29_141058_convert_orders_to_hypertable.php 等依赖此扩展
\connect datahub
CREATE EXTENSION IF NOT EXISTS timescaledb;
-- 可按需追加其他扩展,例如:
-- CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-- CREATE EXTENSION IF NOT EXISTS pg_trgm;
@@ -0,0 +1,26 @@
[Unit]
Description=Datahub Backend (Hyperf)
After=network-online.target datahub-postgres.service datahub-rabbitmq.service
Requires=datahub-postgres.service datahub-rabbitmq.service
[Container]
ContainerName=datahub-backend
Image=localhost/datahub-backend:latest
Network=datahub.network
Volume=/var/container/data/datahub/backend-runtime:/opt/www/runtime:Z
PublishPort=9501:9501
EnvironmentFile=/var/container/data/datahub/env/datahub-backend.env
Secret=datahub-pg-password,type=env,target=DB_PASSWORD
Secret=datahub-rabbitmq-password,type=env,target=AMQP_PASSWORD
Secret=datahub-rabbitmq-password,type=env,target=AMQP_ADMIN_PASSWORD
Secret=datahub-jwt-secret,type=env,target=SIMPLE_JWT_SECRET
Secret=datahub-tools-token,type=env,target=TOOLS_TOKEN
[Service]
Restart=always
TimeoutStartSec=180
[Install]
WantedBy=default.target
@@ -0,0 +1,18 @@
[Unit]
Description=Datahub Frontend (Nginx)
After=network-online.target datahub-backend.service
Requires=datahub-backend.service
[Container]
ContainerName=datahub-frontend
Image=localhost/datahub-frontend:latest
Network=datahub.network
PublishPort=8080:80
EnvironmentFile=/var/container/data/datahub/env/datahub-frontend.env
[Service]
Restart=always
[Install]
WantedBy=default.target
@@ -0,0 +1,31 @@
[Unit]
Description=Datahub PostgreSQL (TimescaleDB)
After=network-online.target
Requires=datahub-network.service
[Container]
ContainerName=datahub-postgres
Image=localhost/datahub-postgres:latest
Network=datahub.network
Volume=/var/container/data/datahub/postgres:/var/lib/postgresql/data:Z,U
PublishPort=127.0.0.1:5416:5432
Environment=POSTGRES_DB=datahub
Environment=POSTGRES_USER=datahub
Environment=POSTGRES_PASSWORD_FILE=/run/secrets/datahub-pg-password
Environment=TZ=Asia/Shanghai
Environment=PGDATA=/var/lib/postgresql/data/pgdata
Secret=datahub-pg-password,type=mount,target=datahub-pg-password,mode=0444
HealthCmd=pg_isready -U datahub -d datahub
HealthInterval=10s
HealthTimeout=5s
HealthRetries=10
[Service]
Restart=always
TimeoutStartSec=180
[Install]
WantedBy=default.target
@@ -0,0 +1,30 @@
[Unit]
Description=Datahub RabbitMQ
After=network-online.target
Requires=datahub-network.service
[Container]
ContainerName=datahub-rabbitmq
Image=docker.io/library/rabbitmq:3-management-alpine
Network=datahub.network
Volume=/var/container/data/datahub/rabbitmq:/var/lib/rabbitmq:Z,U
PublishPort=127.0.0.1:5672:5672
PublishPort=127.0.0.1:15672:15672
Environment=RABBITMQ_DEFAULT_USER=user
Environment=RABBITMQ_DEFAULT_PASS_FILE=/run/secrets/datahub-rabbitmq-password
Environment=RABBITMQ_DEFAULT_VHOST=dataflow
Secret=datahub-rabbitmq-password,type=mount,target=datahub-rabbitmq-password,mode=0444
HealthCmd=rabbitmq-diagnostics -q ping
HealthInterval=15s
HealthTimeout=10s
HealthRetries=10
[Service]
Restart=always
TimeoutStartSec=180
[Install]
WantedBy=default.target
+9
View File
@@ -0,0 +1,9 @@
[Unit]
Description=Datahub internal network
[Network]
NetworkName=datahub
Driver=bridge
[Install]
WantedBy=default.target
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env bash
# 部署环境体检脚本 - 判断阶段 0-1 是否已完成
#
# 用法:bash check-prereqs.sh
#
# 退出码:0 = 全部 OK;1 = 有必须修复的项目(MISS)
# 不开启 set -e:需要继续检查所有项目
B='\033[34m'; G='\033[32m'; Y='\033[33m'; R='\033[31m'; N='\033[0m'
OK_COUNT=0
WARN_COUNT=0
MISS_COUNT=0
FIX_CMDS=()
ok() { echo -e "${G}[OK]${N} $1"; OK_COUNT=$((OK_COUNT+1)); }
warn() { echo -e "${Y}[WARN]${N} $1"; WARN_COUNT=$((WARN_COUNT+1)); [ -n "${2:-}" ] && { echo " 修复: $2"; FIX_CMDS+=("$2"); }; }
miss() { echo -e "${R}[MISS]${N} $1"; MISS_COUNT=$((MISS_COUNT+1)); [ -n "${2:-}" ] && { echo " 修复: $2"; FIX_CMDS+=("$2"); }; }
echo -e "${B}=== datahub 部署环境体检 ===${N}"
echo "用户: $USER ($(id -u))"
echo "主机: $(hostname)"
echo "OS: $(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME" || echo unknown)"
echo
# ---------- 1. 必需的可执行程序 ----------
# podman + 版本
if command -v podman >/dev/null 2>&1; then
VER=$(podman --version | awk '{print $3}')
MAJOR=$(echo "$VER" | cut -d. -f1)
MINOR=$(echo "$VER" | cut -d. -f2)
if [[ "$MAJOR" =~ ^[0-9]+$ ]] && { (( MAJOR > 4 )) || { (( MAJOR == 4 )) && (( MINOR >= 4 )); }; }; then
ok "podman v$VER (≥ 4.4 支持 Quadlet)"
else
miss "podman v$VER 版本过旧(需 ≥ 4.4" "升级 podman 到 4.4+Ubuntu 22.04 用 22.10 PPA,或装 podman-stable"
fi
else
miss "podman 未安装" "sudo apt install -y podman"
fi
# systemctl
command -v systemctl >/dev/null 2>&1 \
&& ok "systemctl 可用" \
|| miss "systemctl 不可用(非 systemd 系统?)"
# systemctl --user 是否可用(rootless 关键)
if systemctl --user list-units >/dev/null 2>&1; then
ok "systemctl --user 可用(rootless 必需)"
else
miss "systemctl --user 不可用" "确保用 SSH 直连登录(非 sudo su 切来),并且 XDG_RUNTIME_DIR=/run/user/\$(id -u) 已设置"
fi
# uidmap
command -v newuidmap >/dev/null 2>&1 \
&& ok "uidmap (newuidmap) 已安装" \
|| miss "uidmap 未安装(rootless 必需)" "sudo apt install -y uidmap"
# slirp4netns
command -v slirp4netns >/dev/null 2>&1 \
&& ok "slirp4netns 已安装" \
|| miss "slirp4netns 未安装" "sudo apt install -y slirp4netns"
# fuse-overlayfs(推荐)
command -v fuse-overlayfs >/dev/null 2>&1 \
&& ok "fuse-overlayfs 已安装(rootless 存储驱动推荐)" \
|| warn "fuse-overlayfs 未安装(不影响功能,但 rootless 性能更好)" "sudo apt install -y fuse-overlayfs"
# git
command -v git >/dev/null 2>&1 \
&& ok "git $(git --version | awk '{print $3}')" \
|| miss "git 未安装" "sudo apt install -y git"
# ---------- 2. rootless 用户配置 ----------
# subuid
if grep -q "^${USER}:" /etc/subuid 2>/dev/null; then
ok "/etc/subuid: $(grep "^${USER}:" /etc/subuid)"
else
miss "/etc/subuid 缺少 $USER 映射" "sudo usermod --add-subuids 100000-165535 $USER"
fi
# subgid
if grep -q "^${USER}:" /etc/subgid 2>/dev/null; then
ok "/etc/subgid: $(grep "^${USER}:" /etc/subgid)"
else
miss "/etc/subgid 缺少 $USER 映射" "sudo usermod --add-subgids 100000-165535 $USER"
fi
# lingering
if loginctl show-user "$USER" 2>/dev/null | grep -q "Linger=yes"; then
ok "lingering 已启用(登出 / 重启后服务依然运行)"
else
warn "lingering 未启用(用户登出后服务停 + 开机不自启)" "sudo loginctl enable-linger $USER"
fi
# ---------- 3. 数据目录可达性 ----------
if [ -d /var/container/data/datahub ]; then
if [ -w /var/container/data/datahub ]; then
ok "/var/container/data/datahub 已存在且可写"
else
warn "/var/container/data/datahub 存在但当前用户不可写" "sudo chown $USER:$USER /var/container/data/datahub"
fi
else
if sudo -n true 2>/dev/null; then
ok "/var/container/data/datahub 不存在(setup-data-dirs.sh 会创建,sudo 免密可用)"
else
warn "/var/container/data/datahub 不存在(setup-data-dirs.sh 将提示输入 sudo 密码)"
fi
fi
# ---------- 4. 网络端口(仅提示) ----------
for port in 8080 9501; do
if ss -ltn 2>/dev/null | awk '{print $4}' | grep -qE ":${port}\$"; then
warn "端口 $port 已被占用" "lsof -i :$port 查看占用进程"
fi
done
# ---------- 汇总 ----------
echo
echo "────────────────────────────────────"
echo -e "结果: ${G}${OK_COUNT} OK${N} / ${Y}${WARN_COUNT} WARN${N} / ${R}${MISS_COUNT} MISS${N}"
echo
if (( MISS_COUNT > 0 )); then
echo -e "${R}存在必须修复的项目${N},请按上面的'修复:'命令处理后重新运行本脚本。"
if (( ${#FIX_CMDS[@]} > 0 )); then
echo
echo -e "${B}--- 一键复制修复命令 ---${N}"
# 去重
printf '%s\n' "${FIX_CMDS[@]}" | awk '!seen[$0]++'
echo -e "${B}--- end ---${N}"
fi
exit 1
fi
if (( WARN_COUNT > 0 )); then
echo -e "${Y}有警告项${N}(不阻止部署,建议处理)"
echo "可执行 install.sh,但部分功能可能受限(例如 lingering 未启用 → 不自启)"
exit 0
fi
echo -e "${G}环境已就绪,可以执行:${N}"
echo -e " ${B}bash deploy/podman/scripts/install.sh${N}"
exit 0
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env bash
# 交互式生成 env 文件
# 用法:bash configure-env.sh
set -euo pipefail
CONF_DIR=/var/container/data/datahub/env
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd)
if [ ! -d "$CONF_DIR" ]; then
echo "[!] $CONF_DIR 不存在,请先执行: bash $SCRIPT_DIR/setup-data-dirs.sh" >&2
exit 1
fi
# ---------- 1. backend env ----------
BACKEND_ENV="$CONF_DIR/datahub-backend.env"
BACKEND_TPL="$REPO_ROOT/deploy/podman/env/datahub-backend.env.example"
if [ -f "$BACKEND_ENV" ]; then
echo "[=] $BACKEND_ENV 已存在,跳过"
else
cp "$BACKEND_TPL" "$BACKEND_ENV"
chmod 600 "$BACKEND_ENV"
echo "[+] 已生成 $BACKEND_ENV(如需调整 TOOLS_HOST 等参数请手动编辑)"
fi
# ---------- 2. frontend env:交互式询问 API_BASE_URL ----------
FRONTEND_ENV="$CONF_DIR/datahub-frontend.env"
CURRENT_VAL=""
if [ -f "$FRONTEND_ENV" ]; then
CURRENT_VAL=$(grep -E "^API_BASE_URL=" "$FRONTEND_ENV" | sed 's/^API_BASE_URL=//')
echo "[=] 当前 $FRONTEND_ENV 中 API_BASE_URL=$CURRENT_VAL"
fi
DEFAULT_IP=$(hostname -I 2>/dev/null | awk '{print $1}')
DEFAULT_URL="http://${DEFAULT_IP:-127.0.0.1}:9501"
SUGGEST="${CURRENT_VAL:-$DEFAULT_URL}"
echo
echo "请输入浏览器访问后端 API 的完整地址。"
echo "示例:http://192.168.1.100:9501 或 https://api.example.com"
read -rp "API_BASE_URL [$SUGGEST]: " INPUT
API_BASE_URL="${INPUT:-$SUGGEST}"
if ! [[ "$API_BASE_URL" =~ ^https?:// ]]; then
echo "[!] 地址必须以 http:// 或 https:// 开头" >&2
exit 1
fi
cat > "$FRONTEND_ENV" <<EOF
# 由 configure-env.sh 生成于 $(date -Iseconds)
API_BASE_URL=$API_BASE_URL
EOF
chmod 600 "$FRONTEND_ENV"
echo
echo "[+] 已写入 $FRONTEND_ENV"
echo " API_BASE_URL=$API_BASE_URL"
echo
echo "下一步:"
echo " 1. 创建/更新 podman secretsbash $SCRIPT_DIR/create-secrets.sh"
echo " 2. 构建镜像后启动 datahub-frontend.service(容器启动时会自动注入此 URL)"
echo " 3. 已运行的服务若要换 IP:编辑 $FRONTEND_ENV 后 systemctl --user restart datahub-frontend.service"
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
# 交互式创建 4 个 podman secret,供 Quadlet 单元引用
# 用法:bash create-secrets.sh
set -euo pipefail
if ! command -v podman >/dev/null 2>&1; then
echo "未检测到 podman,请先安装" >&2
exit 1
fi
create_secret() {
local name=$1
local prompt=$2
local default_cmd=${3:-}
if podman secret exists "$name" 2>/dev/null; then
read -rp "secret [$name] 已存在,是否替换?(y/N): " ans
if [[ "${ans,,}" != "y" ]]; then
echo " 跳过 $name"
return
fi
podman secret rm "$name" >/dev/null
fi
local value
if [[ -n "$default_cmd" ]]; then
read -rp "$prompt(直接回车自动生成): " -s value
echo
if [[ -z "$value" ]]; then
value=$(eval "$default_cmd")
echo " 已自动生成"
fi
else
read -rp "$prompt: " -s value
echo
if [[ -z "$value" ]]; then
echo " 值不能为空" >&2
exit 1
fi
fi
printf '%s' "$value" | podman secret create "$name" -
echo " ✓ 创建 $name"
}
echo "=== 创建 datahub podman secrets ==="
echo
create_secret datahub-pg-password "PostgreSQL datahub 用户密码"
create_secret datahub-rabbitmq-password "RabbitMQ user 用户密码"
create_secret datahub-jwt-secret "JWT 签名 secret" "openssl rand -hex 32"
create_secret datahub-tools-token "TOOLS_TOKEN(外部 store-api 鉴权 token"
echo
echo "完成。当前 secrets"
podman secret ls
+144
View File
@@ -0,0 +1,144 @@
#!/usr/bin/env bash
# datahub 一键部署脚本(阶段 3-7)
#
# 假设阶段 0-2 已完成:podman 已装 / subuid 已配 / lingering 已开 / 代码已 clone
#
# 用法:bash deploy/podman/scripts/install.sh
#
# 本脚本是幂等的:可重复运行;已存在的资源会跳过或询问。
set -euo pipefail
# ---------- 输出辅助 ----------
B='\033[34m'; G='\033[32m'; Y='\033[33m'; R='\033[31m'; N='\033[0m'
step() { echo -e "\n${B}==> [$1/5] $2${N}"; }
ok() { echo -e "${G}[OK]${N} $*"; }
warn() { echo -e "${Y}[!]${N} $*"; }
err() { echo -e "${R}[ERR]${N} $*" >&2; }
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd)
QUADLET_DIR="$HOME/.config/containers/systemd"
# ---------- 前置检查 ----------
echo -e "${B}=== datahub 部署脚本 ===${N}"
echo "工作目录: $REPO_ROOT"
echo
# 调用体检脚本;MISS 不为 0 时退出码 1 → 整个安装流程中止
if ! bash "$SCRIPT_DIR/check-prereqs.sh"; then
err "环境体检未通过,请按上面提示处理后重试"
exit 1
fi
echo
# ---------- 1. 数据目录 ----------
step 1 "创建数据目录"
bash "$SCRIPT_DIR/setup-data-dirs.sh"
# ---------- 2. env 文件 ----------
step 2 "生成 env 文件(交互)"
bash "$SCRIPT_DIR/configure-env.sh"
# ---------- 3. secrets ----------
step 3 "创建 podman secrets(交互)"
bash "$SCRIPT_DIR/create-secrets.sh"
# ---------- 4. 构建镜像 ----------
step 4 "构建 3 个镜像"
cd "$REPO_ROOT"
echo -e "${B} -> datahub-postgres${N}"
podman build -t localhost/datahub-postgres:latest \
-f deploy/podman/images/postgres/Dockerfile .
echo -e "${B} -> datahub-backend${N}"
podman build -t localhost/datahub-backend:latest \
-f backend/Dockerfile backend/
echo -e "${B} -> datahub-frontend${N}"
podman build -t localhost/datahub-frontend:latest \
-f deploy/podman/images/frontend/Dockerfile .
ok "镜像构建完成"
podman images --format " {{.Repository}}:{{.Tag}} ({{.Size}})" | grep "^ localhost/datahub-" || true
# ---------- 5. Quadlet + 启动 ----------
step 5 "部署 Quadlet 单元并启动服务"
mkdir -p "$QUADLET_DIR"
cp "$REPO_ROOT"/deploy/podman/quadlet/* "$QUADLET_DIR/"
ok "Quadlet 单元已复制到 $QUADLET_DIR"
systemctl --user daemon-reload
# 等待容器达到 healthy(仅对配置了 healthcheck 的容器)
wait_healthy() {
local container=$1
local timeout=${2:-90}
local elapsed=0
while (( elapsed < timeout )); do
local status
status=$(podman inspect --format '{{.State.Health.Status}}' "$container" 2>/dev/null || echo "starting")
if [[ "$status" == "healthy" ]]; then
ok " $container healthy${elapsed}s"
return 0
fi
sleep 3
elapsed=$((elapsed + 3))
printf "."
done
echo
err "$container${timeout}s 内未达到 healthy"
return 1
}
# 启动顺序:PG → MQ → Backend → Frontend
echo -e "${B} 启动 datahub-postgres...${N}"
systemctl --user start datahub-postgres.service
wait_healthy datahub-postgres 90 || { journalctl --user -u datahub-postgres.service -n 50 --no-pager; exit 1; }
echo -e "${B} 启动 datahub-rabbitmq...${N}"
systemctl --user start datahub-rabbitmq.service
wait_healthy datahub-rabbitmq 90 || { journalctl --user -u datahub-rabbitmq.service -n 50 --no-pager; exit 1; }
echo -e "${B} 启动 datahub-backend...${N}"
systemctl --user start datahub-backend.service
sleep 5
if ! systemctl --user is-active --quiet datahub-backend.service; then
err "datahub-backend 未运行"
journalctl --user -u datahub-backend.service -n 50 --no-pager
exit 1
fi
ok " datahub-backend 已运行"
echo -e "${B} 启动 datahub-frontend...${N}"
systemctl --user start datahub-frontend.service
sleep 3
if ! systemctl --user is-active --quiet datahub-frontend.service; then
err "datahub-frontend 未运行"
journalctl --user -u datahub-frontend.service -n 50 --no-pager
exit 1
fi
ok " datahub-frontend 已运行"
# ---------- 完成 ----------
HOST_IP=$(hostname -I 2>/dev/null | awk '{print $1}')
echo
echo -e "${G}========================================${N}"
echo -e "${G} 部署完成!${N}"
echo -e "${G}========================================${N}"
echo
echo "服务访问:"
echo " 前端: http://${HOST_IP:-localhost}:8080"
echo " 后端: http://${HOST_IP:-localhost}:9501"
echo " RabbitMQ 控制台: http://${HOST_IP:-localhost}:15672 (账号 user)"
echo
echo "下一步(首次部署必做):"
echo -e " ${Y}podman exec -it datahub-backend php bin/hyperf.php migrate${N}"
echo
echo "常用命令:"
echo " podman ps # 看容器状态"
echo " systemctl --user status datahub-backend # 看服务状态"
echo " journalctl --user -u datahub-backend -f # 看实时日志"
echo " podman exec -it datahub-backend sh # 进容器调试"
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# 创建宿主机数据目录 /var/container/data/datahub/
# 所有运行期产物(数据卷 + env 配置)统一放在此根目录下
#
# 用法:bash setup-data-dirs.sh
set -euo pipefail
DATA_BASE=/var/container/data/datahub
SUBDIRS=(postgres rabbitmq backend-runtime env)
# 1. 父目录需要 root 创建并把所有权移交给当前用户
if [ ! -d "$DATA_BASE" ]; then
echo "[*] $DATA_BASE 不存在,需要 sudo 创建"
sudo mkdir -p "$DATA_BASE"
sudo chown "$USER:$USER" "$DATA_BASE"
fi
# 2. 创建子目录
for d in "${SUBDIRS[@]}"; do
full="$DATA_BASE/$d"
if [ -d "$full" ]; then
echo "[=] $full 已存在,跳过"
else
mkdir -p "$full"
echo "[+] 创建 $full"
fi
done
echo
echo "完成。当前结构:"
ls -la "$DATA_BASE"
echo
echo "说明:"
echo " - postgres / rabbitmq 目录在容器首次启动时由 podman 自动 chown 给容器内用户(:U 标志)"
echo " - backend-runtime 容器以 root 运行,无需额外 chown"
echo " - env/ 存放 datahub-backend.env / datahub-frontend.env(由 configure-env.sh 生成)"
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env bash
# datahub 卸载脚本
#
# 默认行为:停服务 + 移除 Quadlet + 删 secrets,保留数据目录和镜像
#
# 用法:bash uninstall.sh [选项]
# --purge-images 同时删除 3 个本地镜像
# --purge-data 同时删除 /var/container/data/datahub/ 数据目录(危险)
# -y, --yes 不询问确认(用于自动化)
set -euo pipefail
B='\033[34m'; G='\033[32m'; Y='\033[33m'; R='\033[31m'; N='\033[0m'
ok() { echo -e "${G}[OK]${N} $*"; }
warn() { echo -e "${Y}[!]${N} $*"; }
PURGE_IMAGES=false
PURGE_DATA=false
ASSUME_YES=false
for arg in "$@"; do
case "$arg" in
--purge-images) PURGE_IMAGES=true ;;
--purge-data) PURGE_DATA=true ;;
-y|--yes) ASSUME_YES=true ;;
-h|--help)
sed -n '2,11p' "$0"
exit 0
;;
*) echo "未知参数: $arg" >&2; exit 1 ;;
esac
done
QUADLET_DIR="$HOME/.config/containers/systemd"
DATA_DIR=/var/container/data/datahub
SERVICES=(datahub-frontend datahub-backend datahub-rabbitmq datahub-postgres)
QUADLET_FILES=(
datahub-frontend.container
datahub-backend.container
datahub-rabbitmq.container
datahub-postgres.container
datahub.network
)
SECRETS=(datahub-pg-password datahub-rabbitmq-password datahub-jwt-secret datahub-tools-token)
IMAGES=(localhost/datahub-postgres:latest localhost/datahub-backend:latest localhost/datahub-frontend:latest)
echo -e "${B}=== datahub 卸载脚本 ===${N}"
echo
echo "将执行以下操作:"
echo " - 停止 4 个 systemd 服务"
echo " - 移除 ${#QUADLET_FILES[@]} 个 Quadlet 单元"
echo " - 删除 ${#SECRETS[@]} 个 podman secrets"
echo -n " - 镜像: "
[ "$PURGE_IMAGES" = true ] && echo -e "${R}删除${N}" || echo "保留"
echo -n " - 数据目录 $DATA_DIR: "
[ "$PURGE_DATA" = true ] && echo -e "${R}删除(数据将丢失)${N}" || echo "保留"
echo
if [ "$ASSUME_YES" = false ]; then
read -rp "确认继续?(y/N): " ans
[[ "${ans,,}" == "y" ]] || { echo "已取消"; exit 0; }
fi
# 1. 停服务
echo -e "\n${B}-> 停止服务${N}"
for svc in "${SERVICES[@]}"; do
if systemctl --user is-active --quiet "$svc.service" 2>/dev/null; then
systemctl --user stop "$svc.service" || true
echo " 停止 $svc"
fi
done
# 2. 移除 Quadlet 文件
echo -e "\n${B}-> 移除 Quadlet 单元${N}"
for f in "${QUADLET_FILES[@]}"; do
if [ -f "$QUADLET_DIR/$f" ]; then
rm -f "$QUADLET_DIR/$f"
echo " 删除 $f"
fi
done
systemctl --user daemon-reload
# 3. 清理残留容器(保险起见)
for c in datahub-postgres datahub-rabbitmq datahub-backend datahub-frontend; do
if podman container exists "$c" 2>/dev/null; then
podman rm -f "$c" >/dev/null
echo " 删除残留容器 $c"
fi
done
# 4. 删 secrets
echo -e "\n${B}-> 删除 secrets${N}"
for s in "${SECRETS[@]}"; do
if podman secret exists "$s" 2>/dev/null; then
podman secret rm "$s" >/dev/null
echo " 删除 $s"
fi
done
# 5. 删镜像(可选)
if [ "$PURGE_IMAGES" = true ]; then
echo -e "\n${B}-> 删除镜像${N}"
for img in "${IMAGES[@]}"; do
if podman image exists "$img" 2>/dev/null; then
podman rmi "$img" >/dev/null && echo " 删除 $img"
fi
done
fi
# 6. 删数据目录(危险,双重确认)
if [ "$PURGE_DATA" = true ]; then
echo -e "\n${R}-> 删除数据目录${N}"
if [ "$ASSUME_YES" = false ]; then
echo -e "${R}警告:将永久删除 $DATA_DIR 及全部数据${N}"
read -rp "请输入完整的 'yes' 以确认: " ans
if [ "$ans" != "yes" ]; then
echo "数据目录保留"
ASSUME_YES=skip_data
fi
fi
if [ "$ASSUME_YES" != "skip_data" ]; then
sudo rm -rf "$DATA_DIR"
echo " 数据目录已删除"
fi
fi
echo
ok "卸载完成"
[ "$PURGE_DATA" = false ] && warn "数据保留在 $DATA_DIR(重新部署可直接恢复)"
[ "$PURGE_IMAGES" = false ] && warn "镜像保留(podman images | grep datahub 可见)"
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env bash
# datahub 更新脚本:git pull + 智能重建 + 滚动重启
#
# 用法:bash update.sh [选项]
# --force-rebuild 不看 git diff,强制重建所有镜像
# --skip-migrate 检测到新迁移时不提示,直接跳过
set -euo pipefail
B='\033[34m'; G='\033[32m'; Y='\033[33m'; R='\033[31m'; N='\033[0m'
step() { echo -e "\n${B}=> $*${N}"; }
ok() { echo -e "${G}[OK]${N} $*"; }
warn() { echo -e "${Y}[!]${N} $*"; }
err() { echo -e "${R}[ERR]${N} $*" >&2; }
FORCE_REBUILD=false
SKIP_MIGRATE=false
for arg in "$@"; do
case "$arg" in
--force-rebuild) FORCE_REBUILD=true ;;
--skip-migrate) SKIP_MIGRATE=true ;;
-h|--help)
sed -n '2,7p' "$0"
exit 0
;;
*) echo "未知参数: $arg" >&2; exit 1 ;;
esac
done
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd)
QUADLET_DIR="$HOME/.config/containers/systemd"
cd "$REPO_ROOT"
# 1. 必须是 git 仓库且工作区干净
if [ ! -d .git ]; then
err "$REPO_ROOT 不是 git 仓库"
exit 1
fi
if ! git diff --quiet || ! git diff --cached --quiet; then
err "工作区有未提交的变更,先 stash 或 commit"
git status --short
exit 1
fi
# 2. git pull
step "git pull"
OLD_HEAD=$(git rev-parse HEAD)
git pull --ff-only
NEW_HEAD=$(git rev-parse HEAD)
if [ "$OLD_HEAD" = "$NEW_HEAD" ] && [ "$FORCE_REBUILD" = false ]; then
ok "代码已是最新,无需更新(如需强制重建请加 --force-rebuild"
exit 0
fi
# 3. 检测变更范围
if [ "$FORCE_REBUILD" = true ]; then
CH_BACKEND=true; CH_FRONTEND=true; CH_QUADLET=true
CH_PG_IMAGE=true; CH_MIGRATIONS=true
warn "强制重建:忽略 git diff"
else
DIFF=$(git diff --name-only "$OLD_HEAD" "$NEW_HEAD")
matches() { echo "$DIFF" | grep -qE "$1"; }
CH_BACKEND=$(matches '^backend/' && echo true || echo false)
CH_FRONTEND=$(matches '^(frontend/|deploy/podman/images/frontend/)' && echo true || echo false)
CH_QUADLET=$(matches '^deploy/podman/quadlet/' && echo true || echo false)
CH_PG_IMAGE=$(matches '^deploy/podman/images/postgres/' && echo true || echo false)
CH_MIGRATIONS=$(matches '^backend/migrations/' && echo true || echo false)
fi
echo
echo "变更检测:"
printf " %-20s %s\n" "backend image:" "$CH_BACKEND"
printf " %-20s %s\n" "frontend image:" "$CH_FRONTEND"
printf " %-20s %s\n" "postgres image:" "$CH_PG_IMAGE"
printf " %-20s %s\n" "quadlet files:" "$CH_QUADLET"
printf " %-20s %s\n" "migrations:" "$CH_MIGRATIONS"
# 4. 重建镜像
if [ "$CH_PG_IMAGE" = "true" ]; then
step "重建 postgres 镜像"
podman build -t localhost/datahub-postgres:latest \
-f deploy/podman/images/postgres/Dockerfile .
fi
if [ "$CH_BACKEND" = "true" ]; then
step "重建 backend 镜像"
podman build -t localhost/datahub-backend:latest \
-f backend/Dockerfile backend/
fi
if [ "$CH_FRONTEND" = "true" ]; then
step "重建 frontend 镜像"
podman build -t localhost/datahub-frontend:latest \
-f deploy/podman/images/frontend/Dockerfile .
fi
# 5. 同步 Quadlet(任何 quadlet 改动或强制时)
if [ "$CH_QUADLET" = "true" ] || [ "$FORCE_REBUILD" = true ]; then
step "同步 Quadlet 单元"
cp deploy/podman/quadlet/* "$QUADLET_DIR/"
systemctl --user daemon-reload
ok "Quadlet 已更新"
fi
# 6. 滚动重启受影响的服务
step "重启受影响的服务"
restart_svc() {
local svc=$1
echo " 重启 $svc.service"
systemctl --user restart "$svc.service"
sleep 2
if ! systemctl --user is-active --quiet "$svc.service"; then
err "$svc 重启失败"
journalctl --user -u "$svc.service" -n 40 --no-pager
exit 1
fi
}
[ "$CH_PG_IMAGE" = "true" ] && restart_svc datahub-postgres
[ "$CH_BACKEND" = "true" ] && restart_svc datahub-backend
[ "$CH_FRONTEND" = "true" ] && restart_svc datahub-frontend
# Quadlet 改了但镜像没改时,仍需重启对应服务
if [ "$CH_QUADLET" = "true" ]; then
DIFF_Q=$(git diff --name-only "$OLD_HEAD" "$NEW_HEAD" -- deploy/podman/quadlet/ 2>/dev/null || true)
[ "$CH_PG_IMAGE" = "false" ] && echo "$DIFF_Q" | grep -q "datahub-postgres" && restart_svc datahub-postgres
[ "$CH_BACKEND" = "false" ] && echo "$DIFF_Q" | grep -q "datahub-backend" && restart_svc datahub-backend
[ "$CH_FRONTEND" = "false" ] && echo "$DIFF_Q" | grep -q "datahub-frontend" && restart_svc datahub-frontend
echo "$DIFF_Q" | grep -q "datahub-rabbitmq" && restart_svc datahub-rabbitmq
fi
# 7. 迁移提示
if [ "$CH_MIGRATIONS" = "true" ] && [ "$SKIP_MIGRATE" = false ]; then
echo
warn "检测到新迁移文件:"
git diff --name-only "$OLD_HEAD" "$NEW_HEAD" -- backend/migrations/ | sed 's/^/ /'
read -rp "立即执行迁移?(Y/n): " ans
ans="${ans,,}"
if [[ -z "$ans" || "$ans" == "y" ]]; then
podman exec -it datahub-backend php bin/hyperf.php migrate
ok "迁移完成"
else
warn "跳过。后续可手动执行:podman exec -it datahub-backend php bin/hyperf.php migrate"
fi
elif [ "$CH_MIGRATIONS" = "true" ]; then
warn "检测到新迁移但已 --skip-migrate,请手动执行"
fi
echo
ok "更新完成"
echo " $OLD_HEAD"
echo "$NEW_HEAD"
+575 -582
View File
File diff suppressed because it is too large Load Diff
+25
View File
@@ -0,0 +1,25 @@
node_modules
dist
dist-ssr
coverage
playwright-report
test-results
e2e
.eslintcache
*.tsbuildinfo
__screenshots__
.vite
.vscode
.idea
*.suo
*.sw?
*.local
.DS_Store
.git
.gitignore
README.md
+1
View File
@@ -39,6 +39,7 @@ coverage
# Vitest
__screenshots__/
node_modules/.vite/
test-results/
playwright-report/
+41
View File
@@ -0,0 +1,41 @@
# ============================================================
# Stage 1: builder
# ============================================================
FROM docker.io/library/node:22-alpine AS builder
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci --no-audit --no-fund
COPY . .
ARG VITE_API_BASE_URL
ENV VITE_API_BASE_URL=${VITE_API_BASE_URL}
RUN npm run build
# ============================================================
# Stage 2: runtime
# ============================================================
FROM docker.io/library/nginx:1.27-alpine
LABEL org.opencontainers.image.title="datahub-frontend" \
org.opencontainers.image.vendor="WPIC" \
org.opencontainers.image.licenses="MIT" \
org.opencontainers.image.source="https://192.168.30.181:3000/wpic-dev/datahub"
ARG TIMEZONE=Asia/Shanghai
ENV TIMEZONE=${TIMEZONE}
RUN apk add --no-cache tzdata && \
ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime && \
echo "${TIMEZONE}" > /etc/timezone
COPY --from=builder /app/dist /usr/share/nginx/html
COPY nginx.conf /etc/nginx/conf.d/default.conf
EXPOSE 80
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
CMD wget -qO- http://127.0.0.1/ -O /dev/null || exit 1
+15
View File
@@ -0,0 +1,15 @@
import { test, expect } from '@playwright/test'
test('homepage loads and SPA bootstraps', async ({ page }) => {
const response = await page.goto('/')
expect(response, 'page.goto should return a response').not.toBeNull()
expect(response!.status(), 'response status < 500').toBeLessThan(500)
await page.waitForLoadState('domcontentloaded')
const title = await page.title()
expect(title.length, 'page title is non-empty').toBeGreaterThan(0)
const appRoot = page.locator('#app')
await expect(appRoot, '#app root mounts').toBeAttached()
})
-8
View File
@@ -1,8 +0,0 @@
import { test, expect } from '@playwright/test';
// See here how to get started:
// https://playwright.dev/docs/intro
test('visits the app root url', async ({ page }) => {
await page.goto('/');
await expect(page.locator('h1')).toHaveText('You did it!');
})
+1
View File
@@ -8,6 +8,7 @@
</head>
<body>
<div id="app"></div>
<script src="/config.js"></script>
<script type="module" src="/src/main.ts"></script>
</body>
</html>
+36
View File
@@ -0,0 +1,36 @@
server {
listen 80;
server_name _;
root /usr/share/nginx/html;
index index.html;
gzip on;
gzip_types text/plain text/css application/json application/javascript
text/xml application/xml application/xml+rss text/javascript
image/svg+xml;
gzip_min_length 1024;
# API 反代到 backend Swoole HTTP server
# 依赖 datahub.network + ContainerName=datahub-backend (Round 06 quadlet 落实)
location /api/ {
proxy_pass http://datahub-backend:9501;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 60s;
proxy_connect_timeout 10s;
}
location ~* \.(?:css|js|woff2?|ttf|eot|svg|png|jpg|jpeg|gif|ico|webp)$ {
expires 7d;
add_header Cache-Control "public, immutable";
try_files $uri =404;
}
location / {
try_files $uri $uri/ /index.html;
}
}
+8514
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -18,7 +18,7 @@
"format": "prettier --write src/"
},
"dependencies": {
"@antv/g2plot": "^2.4.35",
"@antv/g2plot": "^2.3.32",
"@tailwindcss/vite": "^4.1.16",
"ant-design-vue": "^4.2.6",
"dayjs": "^1.11.20",
@@ -28,7 +28,7 @@
"vue-router": "^4.6.3"
},
"devDependencies": {
"@playwright/test": "^1.56.1",
"@playwright/test": "^1.60.0",
"@tsconfig/node22": "^22.0.2",
"@types/jsdom": "^27.0.0",
"@types/node": "^22.18.11",
+6
View File
@@ -101,6 +101,12 @@ export default defineConfig({
/**
* Use the dev server by default for faster feedback loop.
* Use the preview server on CI for more realistic testing.
*
* CI mode assumes `dist/` is already built by an upstream node container
* (Step 1 of the two-step e2e flow); preview only serves dist here, never
* builds. See docs/tmp/deploy-ref/ci-cd/02-frontend-image/design.md §7.0.1
* for the two-container workflow contract.
*
* Playwright will re-use the local server if there is already a dev-server running.
*/
command: process.env.CI ? 'npm run preview' : 'npm run dev',
+8
View File
@@ -0,0 +1,8 @@
// 运行期注入的配置文件
// Vite 构建时此文件原样拷贝到 dist/;nginx 容器启动时由 docker-entrypoint.d/40-inject-config.sh
// 将 __API_BASE_URL__ 替换为真实地址(来自 API_BASE_URL 环境变量)
//
// 本地 dev 模式占位符不会被替换,request.ts 会自动 fallback 到 VITE_API_BASE_URL
window.__APP_CONFIG__ = {
apiBaseUrl: '__API_BASE_URL__',
};
@@ -2,11 +2,12 @@ import { describe, it, expect } from 'vitest'
import { ADMIN_ONLY_PATH_PREFIXES, isAdminOnlyPath } from '../permissions'
describe('ADMIN_ONLY_PATH_PREFIXES', () => {
it('contains all 7 admin-only paths', () => {
expect(ADMIN_ONLY_PATH_PREFIXES).toHaveLength(7)
it('contains all 8 admin-only paths', () => {
expect(ADMIN_ONLY_PATH_PREFIXES).toHaveLength(8)
expect(ADMIN_ONLY_PATH_PREFIXES).toContain('/users')
expect(ADMIN_ONLY_PATH_PREFIXES).toContain('/roles')
expect(ADMIN_ONLY_PATH_PREFIXES).toContain('/route-groups')
expect(ADMIN_ONLY_PATH_PREFIXES).toContain('/api-keys')
expect(ADMIN_ONLY_PATH_PREFIXES).toContain('/mq-status')
expect(ADMIN_ONLY_PATH_PREFIXES).toContain('/failed-messages')
expect(ADMIN_ONLY_PATH_PREFIXES).toContain('/logs/requests')
+1
View File
@@ -8,6 +8,7 @@ export const ADMIN_ONLY_PATH_PREFIXES: readonly string[] = [
'/users',
'/roles',
'/route-groups',
'/api-keys',
'/mq-status',
'/failed-messages',
'/logs/requests',
@@ -108,7 +108,8 @@ describe('AdminApiKeyPage', () => {
it('重置按钮调用 resetFilters 并 fetchAllKeys', async () => {
await mountPage()
const store = useAdminApiKeyStore()
store.filters.user_id = 5
store.filters.username = 'testuser'
store.filters.email = 'test@example.com'
vi.mocked(api.get).mockClear()
setupApi([])
@@ -119,7 +120,8 @@ describe('AdminApiKeyPage', () => {
btn?.click()
await flushPromises()
expect(store.filters.user_id).toBeUndefined()
expect(store.filters.username).toBeUndefined()
expect(store.filters.email).toBeUndefined()
expect(vi.mocked(api.get)).toHaveBeenCalled()
})
+17 -5
View File
@@ -62,12 +62,19 @@ function handleDelete(id: number) {
<!-- 筛选区 -->
<div class="flex gap-3 mb-4 flex-wrap">
<a-input-number
v-model:value="store.filters.user_id"
placeholder="用户 ID"
:min="1"
style="width: 140px"
<a-input
v-model:value="store.filters.username"
placeholder="用户"
style="width: 150px"
allow-clear
@press-enter="handleSearch"
/>
<a-input
v-model:value="store.filters.email"
placeholder="邮箱"
style="width: 180px"
allow-clear
@press-enter="handleSearch"
/>
<a-select
v-model:value="enabledFilter"
@@ -100,6 +107,7 @@ function handleDelete(id: number) {
>
<template #bodyCell="{ column, record }">
<template v-if="column.key === 'user'">
<div>
<span>{{ (record as ApiKeyRecord).user?.username ?? record.user_id }}</span>
<a-tooltip
v-if="(record as ApiKeyRecord).user?.api_key_enabled === false"
@@ -107,6 +115,10 @@ function handleDelete(id: number) {
>
<a-tag color="red" class="ml-1">已停用</a-tag>
</a-tooltip>
</div>
<div v-if="(record as ApiKeyRecord).user?.email" class="text-xs text-gray-400">
{{ (record as ApiKeyRecord).user!.email }}
</div>
</template>
<template v-else-if="column.key === 'prefix'">
@@ -157,6 +157,22 @@ describe('useRouteGroupStore', () => {
expect(api.get).toHaveBeenCalledWith('/api/v1/routes', undefined)
})
})
describe('syncRoutes', () => {
it('calls sync API and refreshes groups + routes', async () => {
const store = useRouteGroupStore()
vi.mocked(api.post).mockResolvedValueOnce(undefined)
vi.mocked(api.get)
.mockResolvedValueOnce(mockGroups)
.mockResolvedValueOnce(mockRoutes)
await store.syncRoutes()
expect(api.post).toHaveBeenCalledWith('/api/v1/routes/sync')
expect(api.get).toHaveBeenCalledWith('/api/v1/route-groups')
expect(api.get).toHaveBeenCalledWith('/api/v1/routes', undefined)
})
})
})
// ─── Page 组件测试 ───
@@ -177,6 +193,7 @@ describe('RouteGroupsPage', () => {
EditOutlined: { template: '<span class="icon-edit" />' },
DeleteOutlined: { template: '<span class="icon-delete" />' },
SaveOutlined: { template: '<span class="icon-save" />' },
SyncOutlined: { template: '<span class="icon-sync" />' },
},
},
})
@@ -242,6 +259,23 @@ describe('RouteGroupsPage', () => {
expect(wrapper.text()).toContain('未分组')
})
it('renders 同步路由 button and calls syncRoutes on click', async () => {
await mountPage()
const syncBtn = wrapper.findAll('.ant-btn').find((b) => b.text().includes('同步路由'))
expect(syncBtn).toBeDefined()
vi.mocked(api.post).mockResolvedValueOnce(undefined)
vi.mocked(api.get)
.mockResolvedValueOnce(mockGroups)
.mockResolvedValueOnce(mockRoutes)
await syncBtn!.trigger('click')
await flushPromises()
expect(api.post).toHaveBeenCalledWith('/api/v1/routes/sync')
})
it('calls batchAssignRoutes on save button click', async () => {
await mountPage()
+20
View File
@@ -6,6 +6,7 @@ import {
EditOutlined,
DeleteOutlined,
SaveOutlined,
SyncOutlined,
} from '@ant-design/icons-vue'
const groupStore = useRouteGroupStore()
@@ -55,6 +56,7 @@ const localCheckedKeys = ref<{ checked: (string | number)[], halfChecked: (strin
halfChecked: [],
})
const assignSaving = ref(false)
const syncing = ref(false)
// 当前选中组的名称
const selectedGroupName = computed(() => {
@@ -148,6 +150,19 @@ async function handleDelete(group: RouteGroupRecord) {
}
}
async function handleSyncRoutes() {
syncing.value = true
try {
await groupStore.syncRoutes()
message.success('路由同步成功')
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : '同步失败'
message.error(msg)
} finally {
syncing.value = false
}
}
async function handleGroupChange(routeId: number, groupId: number | null) {
try {
await groupStore.assignRouteToGroup(routeId, groupId)
@@ -171,9 +186,14 @@ onMounted(async () => {
<a-col :span="8">
<a-card title="路由组" :loading="groupStore.loading">
<template #extra>
<a-space>
<a-button size="small" :loading="syncing" @click="handleSyncRoutes">
<SyncOutlined /> 同步路由
</a-button>
<a-button type="primary" size="small" @click="openCreateModal">
<PlusOutlined /> 新建
</a-button>
</a-space>
</template>
<!-- 未分组入口 -->
@@ -30,9 +30,10 @@ import { api } from '@/utils/request'
const mockMappings = {
items: [
{ id: 1, company_id: 3, platform_id: 1, store_id: null, origin_sku: '0032', origin_sku_id: 1, platform_outer_sku: 'AMZ-0032', platform_product_id: 'ITEM-001', enabled: true, note: null, created_at: '2026-04-01T00:00:00', updated_at: '2026-04-01T00:00:00' },
{ id: 1, company_id: 3, platform_id: 1, store_id: null, origin_sku: '0032', origin_sku_id: 1, platform_outer_sku: 'AMZ-0032', platform_product_id: 'ITEM-001', enabled: true, bundled: true, note: null, created_at: '2026-04-01T00:00:00', updated_at: '2026-04-01T00:00:00' },
{ id: 2, company_id: 3, platform_id: 1, store_id: null, origin_sku: '0033', origin_sku_id: 2, platform_outer_sku: 'AMZ-0033', platform_product_id: 'ITEM-002', enabled: true, bundled: false, note: null, created_at: '2026-04-01T00:00:00', updated_at: '2026-04-01T00:00:00' },
],
total: 1,
total: 2,
page: 1,
per_page: 15,
}
@@ -148,4 +149,70 @@ describe('SkuMappingsPage', () => {
expect(buttonTexts.some((t) => t.includes('搜索'))).toBe(true)
expect(buttonTexts.some((t) => t.includes('重置'))).toBe(true)
})
it('P7: renders 组合商品 column with blue/default tags per bundled', async () => {
await mountPage()
const html = wrapper.html()
expect(html).toContain('组合商品')
const rows = wrapper.findAll('tbody tr')
const bundledRow = rows.find((r) => r.text().includes('AMZ-0032'))
const plainRow = rows.find((r) => r.text().includes('AMZ-0033'))
expect(bundledRow?.html()).toMatch(/ant-tag-blue[^>]*>\s*是/)
expect(plainRow?.html()).toMatch(/ant-tag[^>]*>\s*否/)
expect(plainRow?.html()).not.toMatch(/ant-tag-blue[^>]*>\s*否/)
})
it('P8: form contains bundled Switch defaulted to off on create', async () => {
await mountPage()
const newBtn = wrapper.findAll('.ant-btn').find((b) => b.text().includes('新建连接'))
await newBtn?.trigger('click')
await flushPromises()
await nextTick()
const bodyHtml = document.body.innerHTML
expect(bodyHtml).toContain('组合商品')
const switches = document.body.querySelectorAll('.ant-switch')
const bundledSwitch = Array.from(switches).find((s) => {
const label = s.closest('.ant-form-item')?.querySelector('.ant-form-item-label')
return label?.textContent?.includes('组合商品')
})
expect(bundledSwitch).toBeTruthy()
expect(bundledSwitch?.classList.contains('ant-switch-checked')).toBe(false)
})
it('P9: origin_sku_id required validation triggers error on empty submit', async () => {
await mountPage()
const newBtn = wrapper.findAll('.ant-btn').find((b) => b.text().includes('新建连接'))
await newBtn?.trigger('click')
await flushPromises()
await nextTick()
const okBtn = Array.from(document.body.querySelectorAll('.ant-modal-footer .ant-btn')).find(
(b) => {
const t = (b as HTMLElement).textContent ?? ''
return t.includes('确 定') || t.includes('确定') || t.includes('OK')
},
) as HTMLElement | undefined
okBtn?.click()
await flushPromises()
await nextTick()
await vi.waitFor(
() => {
expect(document.body.innerHTML).toContain('请选择内部 SKU')
},
{ timeout: 8000, interval: 100 },
)
}, 15000)
it('P10: filter area exposes bundled dropdown with 是/否 options', async () => {
await mountPage()
const formItemLabels = wrapper.findAll('.filter-form .ant-form-item-label label')
const bundledLabel = formItemLabels.find((l) => l.text().includes('组合商品'))
expect(bundledLabel).toBeTruthy()
})
})
+37 -2
View File
@@ -26,6 +26,7 @@ const columns = [
{ title: '内部 SKU', dataIndex: 'origin_sku', width: 140 },
{ title: '平台 SKU', dataIndex: 'platform_outer_sku', width: 160 },
{ title: '平台商品ID', dataIndex: 'platform_product_id', width: 160, ellipsis: true },
{ title: '组合商品', key: 'bundled', width: 100 },
{ title: '状态', key: 'enabled', width: 80 },
{ title: '更新时间', key: 'updated_at', width: 170 },
{ title: '操作', key: 'action', width: 200, fixed: 'right' as const },
@@ -49,6 +50,7 @@ const defaultForm = (): SkuMappingForm => ({
generation_strategy: 'prefix',
warehouse_id: undefined,
enabled: true,
bundled: false,
note: '',
})
@@ -57,7 +59,9 @@ const form = reactive<SkuMappingForm>(defaultForm())
const rules: Record<string, Rule[]> = {
company_id: [{ required: true, message: '请选择公司', trigger: 'change' }],
platform_id: [{ required: true, message: '请选择平台', trigger: 'change' }],
origin_sku: [{ required: true, message: '请输入内部 SKU', trigger: 'blur' }],
origin_sku_id: [
{ required: true, message: '请选择内部 SKU', trigger: 'change', type: 'number' },
],
platform_outer_sku: [{ required: true, message: '请输入或生成平台 SKU', trigger: 'blur' }],
}
@@ -175,6 +179,7 @@ function openEdit(record: SkuMappingForm & { id: number }) {
generation_strategy: record.generation_strategy || 'prefix',
warehouse_id: record.warehouse_id || undefined,
enabled: record.enabled ?? true,
bundled: record.bundled ?? false,
note: record.note || '',
})
nextTick(() => { skipCompanyWatch = false })
@@ -338,6 +343,18 @@ watch(
<a-select-option :value="false">禁用</a-select-option>
</a-select>
</a-form-item>
<a-form-item label="组合商品">
<a-select
:value="(store.filters.bundled as any)"
@update:value="(v: any) => { store.filters.bundled = v }"
placeholder="全部"
allow-clear
style="width: 100px"
>
<a-select-option :value="true"></a-select-option>
<a-select-option :value="false"></a-select-option>
</a-select>
</a-form-item>
<a-form-item>
<a-space>
<a-button type="primary" html-type="submit">
@@ -384,6 +401,11 @@ watch(
</template>
<span v-else class="text-gray-400">平台默认</span>
</template>
<template v-else-if="column.key === 'bundled'">
<a-tag :color="record.bundled ? 'blue' : 'default'">
{{ record.bundled ? '是' : '否' }}
</a-tag>
</template>
<template v-else-if="column.key === 'enabled'">
<a-tag :color="record.enabled ? 'green' : 'default'">
{{ record.enabled ? '启用' : '禁用' }}
@@ -488,7 +510,7 @@ watch(
</a-form-item>
</a-col>
<a-col :span="12">
<a-form-item label="内部 SKU" name="origin_sku">
<a-form-item label="内部 SKU" name="origin_sku_id">
<a-select
v-model:value="form.origin_sku_id"
:placeholder="!form.company_id ? '请先选择公司' : '搜索(最少3个字)或输入SKU'"
@@ -544,6 +566,8 @@ watch(
<a-textarea v-model:value="form.note" placeholder="可选备注" :rows="2" />
</a-form-item>
<a-row :gutter="16">
<a-col :span="12">
<a-form-item label="状态">
<a-switch
v-model:checked="form.enabled"
@@ -551,6 +575,17 @@ watch(
un-checked-children="禁用"
/>
</a-form-item>
</a-col>
<a-col :span="12">
<a-form-item label="组合商品" name="bundled">
<a-switch
v-model:checked="form.bundled"
checked-children=""
un-checked-children=""
/>
</a-form-item>
</a-col>
</a-row>
</a-form>
</a-modal>
@@ -22,7 +22,7 @@ describe('useAdminApiKeyStore', () => {
vi.restoreAllMocks()
})
describe('fetchAllKeys — enabled 查询参数契约', () => {
describe('fetchAllKeys — 查询参数契约', () => {
it('enabled=true 应序列化为 1(后端期望 integer 0/1', async () => {
vi.mocked(api.get).mockResolvedValueOnce(emptyPage)
@@ -60,20 +60,46 @@ describe('useAdminApiKeyStore', () => {
expect.objectContaining({ enabled: undefined }),
)
})
})
describe('toggleUserApiKeyEnabled — 请求体字段名契约', () => {
it('应发送 api_key_enabled 字段(后端 UserController::updateApiKeyEnabled 读取该字段)', async () => {
vi.mocked(api.patch).mockResolvedValueOnce(undefined)
it('username 过滤器应传递给后端', async () => {
vi.mocked(api.get).mockResolvedValueOnce(emptyPage)
const store = useAdminApiKeyStore()
await store.toggleUserApiKeyEnabled(42, true)
store.filters.username = 'testuser'
await store.fetchAllKeys()
expect(api.patch).toHaveBeenCalledWith(
'/api/v1/users/42/api-key-enabled',
{ api_key_enabled: true },
expect(api.get).toHaveBeenCalledWith(
'/api/v1/admin/api-keys',
expect.objectContaining({ username: 'testuser' }),
)
})
it('email 过滤器应传递给后端', async () => {
vi.mocked(api.get).mockResolvedValueOnce(emptyPage)
const store = useAdminApiKeyStore()
store.filters.email = 'test@example.com'
await store.fetchAllKeys()
expect(api.get).toHaveBeenCalledWith(
'/api/v1/admin/api-keys',
expect.objectContaining({ email: 'test@example.com' }),
)
})
it('空字符串过滤器应转换为 undefined', async () => {
vi.mocked(api.get).mockResolvedValueOnce(emptyPage)
const store = useAdminApiKeyStore()
store.filters.username = ''
store.filters.email = ''
await store.fetchAllKeys()
expect(api.get).toHaveBeenCalledWith(
'/api/v1/admin/api-keys',
expect.objectContaining({ username: undefined, email: undefined }),
)
})
})
})
+6 -14
View File
@@ -10,7 +10,8 @@ export const useAdminApiKeyStore = defineStore('admin-api-key', () => {
total: 0,
})
const filters = reactive<AdminApiKeyFilters>({
user_id: undefined,
username: undefined,
email: undefined,
enabled: undefined,
})
@@ -20,7 +21,8 @@ export const useAdminApiKeyStore = defineStore('admin-api-key', () => {
const data = await api.get<PaginatedData<ApiKeyRecord>>('/api/v1/admin/api-keys', {
page: pagination.page,
per_page: pagination.per_page,
user_id: filters.user_id,
username: filters.username || undefined,
email: filters.email || undefined,
enabled: filters.enabled === undefined ? undefined : filters.enabled ? 1 : 0,
})
keys.value = data.items
@@ -55,18 +57,9 @@ export const useAdminApiKeyStore = defineStore('admin-api-key', () => {
}
}
async function toggleUserApiKeyEnabled(userId: number, enabled: boolean) {
try {
await api.patch(`/api/v1/users/${userId}/api-key-enabled`, { api_key_enabled: enabled })
await fetchAllKeys()
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : '操作失败'
message.error(msg)
}
}
function resetFilters() {
filters.user_id = undefined
filters.username = undefined
filters.email = undefined
filters.enabled = undefined
pagination.page = 1
}
@@ -79,7 +72,6 @@ export const useAdminApiKeyStore = defineStore('admin-api-key', () => {
fetchAllKeys,
toggleKey,
deleteKey,
toggleUserApiKeyEnabled,
resetFilters,
}
})
+6
View File
@@ -86,6 +86,11 @@ export const useRouteGroupStore = defineStore('route-group', () => {
await Promise.all([fetchGroups(), fetchRoutes()])
}
async function syncRoutes() {
await api.post('/api/v1/routes/sync')
await Promise.all([fetchGroups(), fetchRoutes()])
}
// 按首段资源名分组,避免同名叶节点与目录共存导致 UI 混乱
const routeTree = computed(() => {
const root: RouteTreeNode[] = []
@@ -167,6 +172,7 @@ export const useRouteGroupStore = defineStore('route-group', () => {
fetchRoutes,
assignRouteToGroup,
batchAssignRoutes,
syncRoutes,
routeTree,
checkedRouteKeys,
}
+8 -2
View File
@@ -9,10 +9,11 @@ export interface SkuMappingRecord {
platform_id: number
store_id: number | null
origin_sku: string
origin_sku_id: number | null
platform_outer_sku: string | null
origin_sku_id: number
platform_outer_sku: string
platform_product_id: string
enabled: boolean
bundled: boolean
note: string | null
created_at: string
updated_at: string
@@ -22,6 +23,7 @@ export interface SkuMappingFilters {
origin_sku: string
platform_outer_sku: string
enabled: boolean | undefined
bundled: boolean | undefined
}
export interface SkuMappingForm {
@@ -35,6 +37,7 @@ export interface SkuMappingForm {
generation_strategy: string
warehouse_id: number | undefined
enabled: boolean
bundled?: boolean
note: string
}
@@ -90,6 +93,7 @@ export const useSkuMappingStore = defineStore('skuMapping', () => {
origin_sku: '',
platform_outer_sku: '',
enabled: undefined,
bundled: undefined,
})
// Lookups
@@ -178,6 +182,7 @@ export const useSkuMappingStore = defineStore('skuMapping', () => {
origin_sku: filters.origin_sku || undefined,
platform_outer_sku: filters.platform_outer_sku || undefined,
enabled: filters.enabled,
bundled: filters.bundled,
})
items.value = data.items
pagination.total = data.total
@@ -196,6 +201,7 @@ export const useSkuMappingStore = defineStore('skuMapping', () => {
filters.origin_sku = ''
filters.platform_outer_sku = ''
filters.enabled = undefined
filters.bundled = undefined
cascadeValue.company_id = undefined
cascadeValue.platform_id = undefined
cascadeValue.store_id = undefined
+3 -2
View File
@@ -315,7 +315,7 @@ export interface ApiKeyRecord {
expires_at: string | null
enabled: boolean
created_at: string
user?: { id: number; username: string; api_key_enabled?: boolean }
user?: { id: number; username: string; email?: string; api_key_enabled?: boolean }
}
export interface ApiKeyCreateParams {
@@ -329,7 +329,8 @@ export interface ApiKeyCreateResult {
}
export interface AdminApiKeyFilters {
user_id: number | undefined
username: string | undefined
email: string | undefined
enabled: boolean | undefined
}
+15 -1
View File
@@ -13,7 +13,21 @@ interface RequestOptions extends RequestInit {
timeout?: number
}
const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || ''
// 运行期配置优先(容器启动时由 nginx entrypoint 注入到 /config.js 写入 window.__APP_CONFIG__
// 构建期 VITE_API_BASE_URL 作为本地 dev 的 fallback
function resolveApiBaseUrl(): string {
if (typeof window !== 'undefined') {
const runtime = (window as unknown as { __APP_CONFIG__?: { apiBaseUrl?: string } }).__APP_CONFIG__
const url = runtime?.apiBaseUrl
// 占位符未替换时(如本地直接打开 public/config.js)忽略
if (url && !(url.startsWith('__') && url.endsWith('__'))) {
return url
}
}
return import.meta.env.VITE_API_BASE_URL || ''
}
const API_BASE_URL = resolveApiBaseUrl()
async function request<T = unknown>(url: string, options: RequestOptions = {}): Promise<T> {
url = `${API_BASE_URL}${url}`
@@ -1 +0,0 @@
{"version":"4.1.4","results":[[":frontend/src/pages/users/__tests__/index.spec.ts",{"duration":0,"failed":true}]]}