Skip to content

B
bigDataSystem
Commit b4ba0b5e authored by chenjiahao's avatar chenjiahao
Browse files
Options

数据库离线加载-源端配置

parent 113ba2c7
Expand all Show whitespace changes
Inline Side-by-side
Showing with 873 additions and 563 deletions
src/views/dataIntegration/dataLoading/dataEntryLake/DeplysModal.vue
View file @ b4ba0b5e
<template>
<BasicModal
style="width: 50%"
v-bind="$attrs"
@register="registerModal"
:title="getTitle"
title="获取元数据"
@ok="handleSubmit"
minHeight="50"
>
<BasicForm @register="registerForm" />
<div style="display: flex">
<BasicTree title="表名" search :treeData="tableTree" />
<BasicForm @register="registerForm" style="width: 70%">
<template #incrementIdentificationColumnAlert>
<Alert
show-icon
style="font-size: 12px"
message="基于数量的加载策略，要求在进行数据加载时，源端的表不要发生数据变更，否则可能会出现数据加载异常。"
type="info"
/>
</template>
</BasicForm>
</div>
</BasicModal>
</template>
<script lang="ts" setup>
import { ref, computed, unref, reactive } from 'vue';
import { BasicModal, useModalInner } from '@/components/Modal';
import { BasicForm, useForm } from '@/components/Form';
import { MoveFormSchema } from './dataEntry.data';
import { Alert } from 'ant-design-vue';
import { ref } from 'vue';
import { useMessage } from '@/hooks/web/useMessage';
import { TreeData } from './mock';
import { tableTreeData } from '@/views/dataIntegration/dataLoading/dataEntryLake/mock';
import { BasicTree } from '@/components/Tree';
import { BasicForm, FormSchema, useForm } from '@/components/Form';
const emit = defineEmits(['success', 'register']);
const emit = defineEmits(['register']);
const { createMessage } = useMessage();
const isUpdate = ref(true);
const rowId = ref('');
const idList = ref([]);
const rowData = ref([]);
let isMove = ref();
//获取接口数据并放在下拉框里（这里是打开了一个弹框）
//初始化表单
const [registerForm, { setFieldsValue, updateSchema, resetFields, validate }] = useForm({
labelWidth: 100,
baseColProps: { span: 24 },
schemas: MoveFormSchema,
showActionButtonGroup: false,
actionColOptions: {
span: 23,
const tableTree = ref(tableTreeData);
const deplysFormSchema: FormSchema[] = [
{
field: 'incrementIdentificationColumn',
label: '增量标识列',
component: 'Input',
componentProps: {
placeholder: '',
},
required: true,
ifShow: false,
},
{
field: 'incrementIdentificationColumnAlert',
component: 'Slot',
slot: 'incrementIdentificationColumnAlert',
ifShow: false,
},
{
field: 'site',
component: 'Checkbox',
label: ' ',
colProps: { lg: 5.5, md: 5.5 },
suffix: '位点',
ifShow: false,
},
{
field: 'divider',
component: 'Divider',
ifShow: false,
},
});
//初始化弹框
const [registerModal, { setModalProps, closeModal }] = useModalInner(async (data) => {
console.log('data', data);
// //每次点击弹窗 需要清空存储的数据
rowData.value = [];
isMove.value = data.isMove;
// //重置表单数据
resetFields();
setModalProps({ confirmLoading: false });
// if (data.idList != null && data.idList.length > 0) {
// /* empty */
// } else {
// rowData.value.push(data.record);
// //单个移动
// rowId.value = data.record.businessId;
// data.record.taskId = data.record.businessId;
// setFieldsValue({
// ...data.record,
// });
// }
const treeList = handleTree(TreeData, 'businessId', undefined, undefined, undefined);
updateSchema([
{
field: 'taskId',
field: 'loadingStrategy',
label: '加载策略',
component: 'RadioGroup',
defaultValue: '基于数量',
componentProps: {
treeData: treeList,
options: [
{ label: '基于数量', value: '基于数量' },
{ label: '基于条件', value: '基于条件' },
],
},
},
]);
console.log('treeList:', treeList);
{
component: 'Divider',
},
];
const [registerForm, { updateSchema }] = useForm({
labelWidth: 90,
labelAlign: 'left',
baseColProps: { lg: 24, md: 24 },
schemas: deplysFormSchema,
showActionButtonGroup: false,
actionColOptions: {
span: 23,
},
});
const getTitle = computed(() => '移动');
//初始化弹框
const [registerModal, { closeModal }] = useModalInner(async (data) => {
tableTree.value = tableTreeData.filter((item) => data.myCheckedKeys.includes(item.key));
const flag = data.loadType !== '全量数据加载';
await updateSchema([{ field: 'incrementIdentificationColumn', ifShow: flag }]);
await updateSchema([{ field: 'incrementIdentificationColumnAlert', ifShow: flag }]);
await updateSchema([{ field: 'site', ifShow: flag }]);
await updateSchema([{ field: 'divider', ifShow: flag }]);
});
/**确定按钮*/
async function handleSubmit() {
// console.log('isMove.value', isMove.value);
if (isMove.value === 1) {
createMessage.success('移动成功！');
} else {
createMessage.success('复制成功！');
}
closeModal();
}
/**数组对象转成树*/
function handleTree(data, id, parentId, children, rootId) {
id = id || 'id';
parentId = parentId || 'parentId';
children = children || 'children';
rootId =
rootId ||
Math.min.apply(
Math,
data.map((item) => {
return item[parentId];
}),
) ||
0;
// 对源数据深度克隆
const cloneData = JSON.parse(JSON.stringify(data));
// 循环所有项
const treeData = cloneData.filter((father) => {
const branchArr = cloneData.filter((child) => {
// 返回每一项的子级数组
return father[id] === child[parentId];
});
branchArr.length > 0 ? (father.children = branchArr) : '';
// 返回第一层
return father[parentId] === rootId;
});
return treeData !== '' ? treeData : data;
}
</script>
<style scoped lang="scss">
.getMetadataInput {
width: 120px;
}
</style>
src/views/dataIntegration/dataLoading/dataEntryLake/getMetadataModal.vue
View file @ b4ba0b5e
......@@ -7,20 +7,26 @@
minHeight="50"
>
<div style="display: flex; justify-content: space-between">
<a-input-search style="width: 220px" />
<a-input-search style="width: 220px" @search="onSearch" />
<a-button @click="handleClick">
通过序号批量选择
<Icon v-if="unfold" icon="fe:arrow-up" />
<Icon v-else icon="fe:arrow-down" />
</a-button>
</div>
<div v-if="unfold">
从<a-input class="getMetadataInput" style="margin-right: 5px" />到<a-input
<div v-if="unfold" style="margin-top: 5px">
从<a-input
v-model:value="startId"
class="getMetadataInput"
style="margin-right: 5px; margin-left: 5px"
style="margin-right: 10px; margin-left: 10px"
/>
<a-button style="margin-right: 5px">选择</a-button>
<a-button style="margin-right: 5px">清空</a-button>
到<a-input
v-model:value="endId"
class="getMetadataInput"
style="margin-right: 10px; margin-left: 10px"
/>
<a-button @click="handleSelect" style="margin-right: 10px">选择</a-button>
<a-button @click="handleResetInputs" style="margin-right: 10px">清空</a-button>
</div>
<BasicTable @register="registerTable" />
</BasicModal>
......@@ -37,9 +43,11 @@
const emit = defineEmits(['success', 'register']);
const { createMessage } = useMessage();
const unfold = ref(false);
let startId = ref();
let endId = ref();
let getMetadataTable = ref(getMetadataTableList);
const [registerTable] = useTable({
const [registerTable, { reload, getDataSource, getRowSelection, setSelectedRowKeys }] = useTable({
api: async () => {
const response = {
pageNu: '1',
......@@ -72,6 +80,57 @@
async function handleSubmit() {
closeModal();
}
function handleSelect() {
const selectedRowKeys = getRowSelection().selectedRowKeys;
const ids = ref([...selectedRowKeys]);
const dataIds = ref(getDataSource());
if (startId.value && endId.value) {
for (let i = startId.value - 1; i <= endId.value - 1 && i < getDataSource().length; i++) {
const businessId = dataIds.value[i].businessId;
if (!ids.value.includes(businessId)) {
ids.value.push(businessId);
}
}
}
setSelectedRowKeys(ids.value);
}
function handleResetInputs() {
const selectedRowKeys = getRowSelection().selectedRowKeys;
const ids = ref([...selectedRowKeys]);
const dataIds = ref(getDataSource());
if (startId.value && endId.value) {
const startIndex = startId.value - 1;
const endIndex = endId.value - 1;
// 过滤掉与 i 相等的值
const filteredIds = ids.value.filter((id) => {
for (let i = startIndex; i <= endIndex && i < getDataSource().length; i++) {
if (dataIds.value[i].businessId === id) {
return false;
}
}
return true;
});
// // 清空 startId 和 endId
// startId.value = null;
// endId.value = null;
setSelectedRowKeys(filteredIds);
}
}
function onSearch(searchValue) {
getMetadataTable.value = getMetadataTableList.filter((item) =>
item.tableName.includes(searchValue),
);
reload();
}
</script>
<style scoped lang="scss">
.getMetadataInput {
......
src/views/dataIntegration/dataLoading/dataEntryLake/mock.ts
View file @ b4ba0b5e
......@@ -206,61 +206,73 @@ export const tableTreeData = [
{
title: 'bm_datasource',
key: '0',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, catalog_id FROM bm_datasource',
},
{
title: 'user_info',
key: '1',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, catalog_id, name, description FROM user_info',
},
{
title: 'customer_details',
key: '2',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, user_id, username, email FROM customer_details',
},
{
title: 'order_history',
key: '3',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, order_id, total_amount FROM order_history',
},
{
title: 'product_inventory',
key: '4',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, product_id, product_name, price FROM product_inventory',
},
{
title: 'transaction_logs',
key: '5',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, category_id, category_name FROM transaction_logs',
},
{
title: 'employee_records',
key: '6',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, customer_id, customer_name FROM employee_records',
},
{
title: 'payment_details',
key: '7',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, customer_id, order_date FROM payment_details',
},
{
title: 'shipping_addresses',
key: '8',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, product_id, stock_quantity, last_updated FROM shipping_addresses',
},
{
title: 'invoice_data',
key: '9',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, user_id, address, phone_number FROM invoice_data',
},
{
title: 'customer_feedback',
key: '10',
icon: 'majesticons:table-line',
sql: 'SELECT uuid, category_id, category_description, parent_category_id FROM customer_feedback',
},
{
title: 'supplier_info',
key: '11',
icon: 'majesticons:table-line',
sql: 'SELECT * FROM supplier_info', // 假设 supplier_info 表没有在 isCustomSQLTableList 中定义字段
},
];
......@@ -496,6 +508,99 @@ export const isCustomSQLTableList = [
},
];
export const isBatchCustomSQLTableList = [
{
businessId: '1',
ownershipTableId: '1',
fieldName: 'uuid',
fieldType: 'VARCHAR(50,0)',
},
{
businessId: '2',
ownershipTableId: '1',
fieldName: 'catalog_id',
fieldType: 'BIGINT(19,0)',
},
{
businessId: '3',
ownershipTableId: '1',
fieldName: 'name',
fieldType: 'VARCHAR(255,0)',
},
{
businessId: '4',
ownershipTableId: '1',
fieldName: 'description',
fieldType: 'TEXT',
},
{
businessId: '5',
ownershipTableId: '2',
fieldName: 'uuid',
fieldType: 'VARCHAR(50,0)',
},
{
businessId: '6',
ownershipTableId: '2',
fieldName: 'user_id',
fieldType: 'BIGINT(19,0)',
},
{
businessId: '7',
ownershipTableId: '2',
fieldName: 'username',
fieldType: 'VARCHAR(255,0)',
},
{
businessId: '8',
ownershipTableId: '2',
fieldName: 'email',
fieldType: 'VARCHAR(255,0)',
},
{
businessId: '9',
ownershipTableId: '3',
fieldName: 'uuid',
fieldType: 'VARCHAR(50,0)',
},
{
businessId: '10',
ownershipTableId: '3',
fieldName: 'order_id',
fieldType: 'BIGINT(19,0)',
},
{
businessId: '11',
ownershipTableId: '3',
fieldName: 'total_amount',
fieldType: 'DECIMAL(10,2)',
},
{
businessId: '12',
ownershipTableId: '4',
fieldName: 'uuid',
fieldType: 'VARCHAR(50,0)',
},
{
businessId: '13',
ownershipTableId: '4',
fieldName: 'product_id',
fieldType: 'BIGINT(19,0)',
},
{
businessId: '14',
ownershipTableId: '4',
fieldName: 'product_name',
fieldType: 'VARCHAR(255,0)',
},
{
businessId: '15',
ownershipTableId: '4',
fieldName: 'price',
fieldType: 'DECIMAL(10,2)',
},
];
export const notCustomSQLTableList = [
// bm_datasource
{
......
src/views/dataIntegration/dataLoading/dataEntryLake/offlineLoading.data.ts
View file @ b4ba0b5e
import { FormSchema } from '@/components/Form';
import { BasicColumn } from '@/components/Table';
export const sourceSideConfigurationFormSchema: FormSchema[] = [
{
field: 'loadType',
label: '加载方式',
component: 'RadioGroup',
defaultValue: '全量数据加载',
componentProps: {
options: [
{ label: '全量数据加载', value: '全量数据加载' },
{ label: '增量数据加载', value: '增量数据加载' },
],
},
},
{
field: 'metadataAcquisitionMode',
label: '元数据获取方式',
component: 'RadioGroup',
defaultValue: '从JDBC获取元数据',
componentProps: ({ formModel, formActionType }) => ({
onChange: () => {
const flag = formModel.metadataAcquisitionMode === '从JDBC获取元数据';
formActionType.updateSchema([{ field: 'dataBase', ifShow: flag, required: flag }]);
formActionType.updateSchema([{ field: 'metadataType', ifShow: flag, required: flag }]);
formActionType.updateSchema([{ field: 'getMetadata', ifShow: flag }]);
},
options: [
{ label: '从JDBC获取元数据', value: '从JDBC获取元数据' },
{ label: '自定义SQL', value: '自定义SQL' },
{ label: '导入元数据文件', value: '导入元数据文件' },
],
}),
},
{
field: 'dataSource',
label: '数据源',
component: 'Select',
required: true,
defaultValue: 'KunOB数据源',
componentProps: {
placeholder: '请选择数据源',
options: [
{ label: 'KunOB数据源', value: 'KunOB数据源' },
{ label: 'MongoDB数据源', value: 'MongoDB数据源' },
{ label: 'AmazonS3', value: 'AmazonS3' },
],
},
},
{
field: 'dataBase',
label: '数据库',
component: 'Cascader',
required: true,
componentProps: {
placeholder: '请选择数据源',
displayRender: ({ labels }) => {
return labels[labels.length - 1];
},
options: [
{
value: '数据库对象资源',
label: '数据库对象资源',
children: [
{
value: '数据中台工作区01',
label: '数据中台工作区01',
children: [
{
value: 'ArgoDB_Dev01',
label: 'ArgoDB_Dev01',
},
{
value: 'ArgoDB_Dev02',
label: 'ArgoDB_Dev02',
},
{
value: 'GbaseDB_Dev01',
label: 'GbaseDB_Dev01',
},
],
},
],
},
],
},
},
{
field: 'metadataType',
label: '元数据类型',
component: 'Input',
required: true,
componentProps: {
placeholder: '以,分隔输入元数据类型',
},
},
{
field: 'getMetadata',
label: ' ',
component: 'Slot',
slot: 'getMetadata',
},
{
field: 'maximumNumberOfDatabaseConnections',
label: '源库最大连接数',
component: 'Input',
componentProps: {
placeholder: '请输入源库最大连接数',
},
},
];
export const LoadingStrategyFormSchema: FormSchema[] = [
{
field: 'loadingStrategy',
label: '加载策略',
component: 'RadioGroup',
defaultValue: '基于数量',
componentProps: ({ formModel, formActionType }) => ({
onChange: () => {
const flag = formModel.loadingStrategy === '基于数量';
formActionType.updateSchema([{ field: 'quantityBasedAlert', ifShow: flag }]);
formActionType.updateSchema([{ field: 'conditionBasedAlert', ifShow: !flag }]);
},
options: [
{ label: '基于数量', value: '基于数量' },
{ label: '基于条件', value: '基于条件' },
],
}),
},
{
field: 'quantityBasedAlert',
component: 'Slot',
slot: 'quantityBasedAlert',
},
{
field: 'conditionBasedAlert',
component: 'Slot',
slot: 'conditionBasedAlert',
ifShow: false,
},
{
component: 'Divider',
},
{
field: 'customSQL',
label: '自定义SQL',
component: 'RadioGroup',
defaultValue: '否',
required: true,
componentProps: ({ formModel, formActionType }) => ({
onChange: () => {
const flag = formModel.customSQL === '否';
formActionType.updateSchema([{ field: 'filterCondition', ifShow: flag }]);
formActionType.updateSchema([{ field: 'notCustomSQLTable', ifShow: flag }]);
formActionType.updateSchema([{ field: 'querySQL', ifShow: !flag }]);
formActionType.updateSchema([{ field: 'parsingSQL', ifShow: !flag }]);
formActionType.updateSchema([{ field: 'isCustomSQLAlert', ifShow: !flag }]);
formActionType.updateSchema([{ field: 'isCustomSQLTable', ifShow: !flag }]);
},
options: [
{ label: '是', value: '是' },
{ label: '否', value: '否' },
],
}),
},
{
field: 'filterCondition',
label: '过滤条件',
component: 'InputTextArea',
componentProps: {
placeholder: '请输入',
},
},
{
field: 'notCustomSQLTable',
component: 'Slot',
slot: 'notCustomSQLTable',
},
{
field: 'querySQL',
label: '查询SQL',
component: 'InputTextArea',
componentProps: {
placeholder: '请输入',
},
ifShow: false,
},
{
field: 'parsingSQL',
label: ' ',
component: 'Slot',
slot: 'parsingSQL',
ifShow: false,
},
{
field: 'isCustomSQLAlert',
component: 'Slot',
slot: 'isCustomSQLAlert',
ifShow: false,
},
{
field: 'isCustomSQLTable',
component: 'Slot',
slot: 'isCustomSQLTable',
ifShow: false,
},
];
export const isCustomSQLColumns: BasicColumn[] = [
{
title: '列名',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment