szz
/
ShiShiYiBan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

【功能优化】 产品发布创建超级表优化

This commit is contained in:
安浩浩 2024-10-26 23:15:31 +08:00
parent ea8dd67e9e
commit 7b5aa23d5c
13 changed files with 491 additions and 371 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
yudao-module-iot/yudao-module-iot-api/src/main/java/cn/iocoder/yudao/module/iot/domain/FieldsVo.java
View File

@ -1,14 +1,18 @@
package cn.iocoder.yudao.module.iot.domain;
import lombok.Builder;
import lombok.Data;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
/**
* 字段信息 VO
*/
@Data
@Builder
public class FieldsVo {
private static final long serialVersionUID = 1L;
/**
* 字段名称
@ -24,24 +28,4 @@ public class FieldsVo {
* 字段字节大小
*/
private Integer size;
public static FieldsVo fieldsTranscoding(Fields fields) throws SQLException {
// if (StringUtils.isBlank(fields.getFieldName()) || fields.getDataType() == null) {
// throw new SQLException("invalid operation: fieldName or dataType can not be null");
// }
// FieldsVo fieldsVo = new FieldsVo();
// fieldsVo.setFieldName(fields.getFieldName());
// fieldsVo.setDataType(fields.getDataType().getDataType());
// fieldsVo.setSize(fields.getSize());
// return fieldsVo;
return null;
}
public static List<FieldsVo> fieldsTranscoding(List<Fields> fieldsList) throws SQLException {
List<FieldsVo> fieldsVoList = new ArrayList<>();
for (Fields fields : fieldsList) {
fieldsVoList.add(fieldsTranscoding(fields));
}
return fieldsVoList;
}
}

44
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/dataobject/tdengine/FieldParser.java
View File

@ -10,7 +10,6 @@ import java.util.stream.Collectors;
/**
* FieldParser 类用于解析和转换物模型字段到 TDengine 字段
*
*/
public class FieldParser {
@ -35,49 +34,58 @@ public class FieldParser {
* @param property 物模型属性
* @return TdField对象
*/
public static TdField parse(ThingModelProperty property) {
public static TdFieldDO parse(ThingModelProperty property) {
String fieldName = property.getIdentifier().toLowerCase();
ThingModelDataType type = property.getDataType();
// 将物模型字段类型映射为td字段类型
String fType = TYPE_MAPPING.get(type.getType().toUpperCase());
int len = -1;
// 如果字段类型为NCHAR，默认长度为64
int dataLength = 0;
if ("NCHAR".equals(fType)) {
len = 64;
dataLength = 64;
}
return new TdField(fieldName, fType, len);
return new TdFieldDO(fieldName, fType, dataLength);
}
/**
* 获取物模型中的字段列表
* 从物模型中获取字段列表
*
* @param thingModel 物模型响应对象
* @return 字段列表
*/
public static List<TdField> parse(ThingModelRespVO thingModel) {
return thingModel.getModel().getProperties().stream().map(FieldParser::parse).collect(Collectors.toList());
public static List<TdFieldDO> parse(ThingModelRespVO thingModel) {
return thingModel.getModel().getProperties().stream()
.map(FieldParser::parse)
.collect(Collectors.toList());
}
/**
* 将从库中查出来的字段信息转换为td字段对象
* 将从库中查出来的字段信息转换为 TDengine 字段对象
*
* @param rows 从库中查出的字段信息列表
* @return 转换后的 TDengine 字段对象列表
*/
public static List<TdField> parse(List<List<Object>> rows) {
public static List<TdFieldDO> parse(List<List<Object>> rows) {
return rows.stream().map(row -> {
String type = row.get(1).toString().toUpperCase();
return new TdField(
int dataLength = "NCHAR".equals(type) ? Integer.parseInt(row.get(2).toString()) : -1;
return new TdFieldDO(
row.get(0).toString(),
type,
type.equals("NCHAR") ? Integer.parseInt(row.get(2).toString()) : -1);
dataLength
);
}).collect(Collectors.toList());
}
/**
* 获取字段字义
*/
public static String getFieldDefine(TdField field) {
return "`" + field.getName() + "`" + " "
+ (field.getLength() > 0 ? String.format("%s(%d)", field.getType(), field.getLength())
: field.getType());
public static String getFieldDefine(TdFieldDO field) {
return "`" + field.getFieldName() + "`" + " "
+ (field.getDataLength() > 0 ? String.format("%s(%d)", field.getDataType(), field.getDataLength())
: field.getDataType());
}
}
}

2
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/dataobject/tdengine/TableData.java
View File

@ -36,4 +36,4 @@ public class TableData {
* 超级表名称
*/
private String superTableName;
}
}

26
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/dataobject/tdengine/TableManager.java
View File

@ -45,7 +45,7 @@ public class TableManager {
/**
* 获取创建表sql
*/
public static String getCreateSTableSql(String tbName, List<TdField> fields, TdField... tags) {
public static String getCreateSTableSql(String tbName, List<TdFieldDO> fields, TdFieldDO... tags) {
if (fields.isEmpty()) {
return null;
}
@ -53,7 +53,7 @@ public class TableManager {
// 生成字段片段
StringBuilder sbField = new StringBuilder("time TIMESTAMP,");
for (TdField field : fields) {
for (TdFieldDO field : fields) {
sbField.append(FieldParser.getFieldDefine(field));
sbField.append(",");
}
@ -63,7 +63,7 @@ public class TableManager {
// 生成tag
StringBuilder sbTag = new StringBuilder();
for (TdField tag : tags) {
for (TdFieldDO tag : tags) {
sbTag.append(FieldParser.getFieldDefine(tag))
.append(",");
}
@ -76,7 +76,7 @@ public class TableManager {
/**
* 获取创建普通表sql
*/
public static String getCreateCTableSql(String tbName, List<TdField> fields) {
public static String getCreateCTableSql(String tbName, List<TdFieldDO> fields) {
if (fields.size() == 0) {
return null;
}
@ -84,7 +84,7 @@ public class TableManager {
//生成字段片段
StringBuilder sbField = new StringBuilder("time timestamp,");
for (TdField field : fields) {
for (TdFieldDO field : fields) {
sbField.append(FieldParser.getFieldDefine(field));
sbField.append(",");
}
@ -116,9 +116,9 @@ public class TableManager {
/**
* 获取添加字段sql
*/
public static String getAddSTableColumnSql(String tbName, List<TdField> fields) {
public static String getAddSTableColumnSql(String tbName, List<TdFieldDO> fields) {
StringBuilder sbAdd = new StringBuilder();
for (TdField field : fields) {
for (TdFieldDO field : fields) {
sbAdd.append(String.format(ALTER_STABLE_ADD_COL_TPL,
tbName,
FieldParser.getFieldDefine(field)
@ -130,9 +130,9 @@ public class TableManager {
/**
* 获取修改字段sql
*/
public static String getModifySTableColumnSql(String tbName, List<TdField> fields) {
public static String getModifySTableColumnSql(String tbName, List<TdFieldDO> fields) {
StringBuilder sbModify = new StringBuilder();
for (TdField field : fields) {
for (TdFieldDO field : fields) {
sbModify.append(String.format(ALTER_STABLE_MODIFY_COL_TPL,
tbName,
FieldParser.getFieldDefine(field)
@ -144,15 +144,15 @@ public class TableManager {
/**
* 获取删除字段sql
*/
public static String getDropSTableColumnSql(String tbName, List<TdField> fields) {
public static String getDropSTableColumnSql(String tbName, List<TdFieldDO> fields) {
StringBuilder sbDrop = new StringBuilder();
for (TdField field : fields) {
for (TdFieldDO field : fields) {
sbDrop.append(String.format(ALTER_STABLE_DROP_COL_TPL,
tbName,
field.getName()
field.getFieldName()
));
}
return sbDrop.toString();
}
}
}

10
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/dataobject/tdengine/TdField.java → yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/dataobject/tdengine/TdFieldDO.java
View File

@ -1,6 +1,7 @@
package cn.iocoder.yudao.module.iot.dal.dataobject.tdengine;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -10,20 +11,21 @@ import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TdField {
@Builder
public class TdFieldDO {
/**
* 字段名称
*/
private String name;
private String fieldName;
/**
* 字段类型
*/
private String type;
private String dataType;
/**
* 字段长度
*/
private int length;
private Integer dataLength = 0;
}

18
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/dataobject/tdengine/TdResponse.java
View File

@ -17,13 +17,25 @@ public class TdResponse {
public static final int CODE_SUCCESS = 0;
public static final int CODE_TB_NOT_EXIST = 866;
/**
* 状态
*/
private String status;
/**
* 错误码
*/
private int code;
/**
* 错误信息
*/
private String desc;
//[["time","TIMESTAMP",8,""],["powerstate","TINYINT",1,""],["brightness","INT",4,""],["deviceid","NCHAR",32,"TAG"]]
private List<Object[]> data;
/**
* 列信息
* [["time","TIMESTAMP",8,""],["powerstate","TINYINT",1,""],["brightness","INT",4,""],["deviceid","NCHAR",32,"TAG"]]
*/
private List data;
}
}

23
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/dataobject/tdengine/TdRestApi.java
View File

@ -14,23 +14,23 @@ import org.springframework.stereotype.Service;
@Service
public class TdRestApi {
@Value("${spring.datasource.dynamic.datasource.master.url}")
@Value("${spring.datasource.dynamic.datasource.tdengine.url}")
private String url;
@Value("${spring.datasource.dynamic.datasource.master.username}")
@Value("${spring.datasource.dynamic.datasource.tdengine.username}")
private String username;
@Value("${spring.datasource.dynamic.datasource.master.password}")
@Value("${spring.datasource.dynamic.datasource.tdengine.password}")
private String password;
/**
* 获取 REST API URL
*/
private String getRestApiUrl() {
//jdbc:TAOS-RS://127.0.0.1:6041/iotkit?xxxx
String restUrl = url.replace("jdbc:TAOS-RS://" , "")
.replaceAll("\\?.*" , "");
// /rest/sql/iotkit
String restUrl = url.replace("jdbc:TAOS-RS://", "")
.replaceAll("\\?.*", "");
int idx = restUrl.lastIndexOf("/");
//127.0.0.1:6041/rest/sql/iotkit
return String.format("%s/rest/sql/%s" , restUrl.substring(0, idx), restUrl.substring(idx + 1));
return String.format("%s/rest/sql/%s", restUrl.substring(0, idx), restUrl.substring(idx + 1));
}
@ -48,11 +48,10 @@ public class TdRestApi {
* 执行sql
*/
public TdResponse execSql(String sql) {
log.info("exec td sql:{}" , sql);
log.info("exec td sql:{}", sql);
HttpRequest request = newApiRequest(sql);
HttpResponse response = request.execute();
return JSONUtil.toBean(response.body(), TdResponse.class);
}
}
}

87
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/dal/tdengine/TdEngineMapper.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.iot.dal.tdengine;
import cn.iocoder.yudao.module.iot.dal.dataobject.tdengine.TdFieldDO;
import cn.iocoder.yudao.module.iot.domain.FieldsVo;
import cn.iocoder.yudao.module.iot.domain.SelectDto;
import cn.iocoder.yudao.module.iot.domain.TableDto;
@ -17,24 +18,80 @@ import java.util.Map;
@DS("tdengine")
public interface TdEngineMapper {
/**
* 创建数据库
*
* @param dataBaseName 数据库名称
*/
@InterceptorIgnore(tenantLine = "true")
void createDatabase(@Param("dataBaseName") String dataBaseName);
void createSuperTable(@Param("schemaFields") List<FieldsVo> schemaFields,
@Param("tagsFields") List<FieldsVo> tagsFields,
/**
* 创建超级表
*
* @param schemaFields schema字段
* @param tagsFields tags字段
* @param dataBaseName 数据库名称
* @param superTableName 超级表名称
*/
@InterceptorIgnore(tenantLine = "true")
void createSuperTable(@Param("schemaFields") List<TdFieldDO> schemaFields,
@Param("tagsFields") List<TdFieldDO> tagsFields,
@Param("dataBaseName") String dataBaseName,
@Param("superTableName") String superTableName);
/**
* 查看超级表 - 显示当前数据库下的所有超级表信息
* SQL：SHOW STABLES [LIKE tb_name_wildcard];
*
* @param dataBaseName 数据库名称
* @param superTableName 超级表名称
*/
@InterceptorIgnore(tenantLine = "true")
List<Map<String, Object>> showSuperTables(@Param("dataBaseName") String dataBaseName,
@Param("superTableName") String superTableName);
/**
* 查看超级表 - 获取超级表的结构信息
* SQL：DESCRIBE [db_name.]stb_name;
* <p>
* * @param dataBaseName 数据库名称
* * @param superTableName 超级表名称
*/
@InterceptorIgnore(tenantLine = "true")
List<Map<String, Object>> describeSuperTable(@Param("dataBaseName") String dataBaseName,
@Param("superTableName") String superTableName);
/**
* 为超级表添加列
*
* @param dataBaseName 数据库名称
* @param superTableName 超级表名称
* @param field 字段信息
*/
@InterceptorIgnore(tenantLine = "true")
void addColumnForSuperTable(@Param("dataBaseName") String dataBaseName,
@Param("superTableName") String superTableName,
@Param("field") TdFieldDO field);
/**
* 为超级表删除列
*
* @param dataBaseName 数据库名称
* @param superTableName 超级表名称
* @param field 字段信息
*/
@InterceptorIgnore(tenantLine = "true")
void dropColumnForSuperTable(@Param("dataBaseName") String dataBaseName,
@Param("superTableName") String superTableName,
@Param("field") TdFieldDO field);
void createTable(TableDto tableDto);
void insertData(TableDto tableDto);
List<Map<String, Object>> selectByTimestamp(SelectDto selectDto);
void addColumnForSuperTable(@Param("superTableName") String superTableName,
@Param("fieldsVo") FieldsVo fieldsVo);
void dropColumnForSuperTable(@Param("superTableName") String superTableName,
@Param("fieldsVo") FieldsVo fieldsVo);
void addTagForSuperTable(@Param("superTableName") String superTableName,
@Param("fieldsVo") FieldsVo fieldsVo);
@ -44,14 +101,6 @@ public interface TdEngineMapper {
Map<String, Long> getCountByTimestamp(SelectDto selectDto);
/**
* 检查表是否存在
*
* @param dataBaseName 数据库名称
* @param tableName 表名称
*/
Integer checkTableExists(@Param("dataBaseName") String dataBaseName, @Param("tableName") String tableName);
Map<String, Object> getLastData(SelectDto selectDto);
List<Map<String, Object>> getHistoryData(SelectVisualDto selectVisualDto);
@ -62,13 +111,5 @@ public interface TdEngineMapper {
List<Map<String, Object>> getLastDataByTags(TagsSelectDao tagsSelectDao);
/**
* 创建超级表
*
* @param sql sql
* @return 返回值
*/
@InterceptorIgnore(tenantLine = "true")
Integer createSuperTableDevice(String sql);
}

282
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/service/tdengine/IotDbStructureDataServiceImpl.java
View File

@ -1,19 +1,22 @@
package cn.iocoder.yudao.module.iot.service.tdengine;
import cn.hutool.json.JSONUtil;
import cn.hutool.core.collection.CollUtil;
import cn.iocoder.yudao.module.iot.controller.admin.thinkmodelfunction.thingModel.ThingModelProperty;
import cn.iocoder.yudao.module.iot.controller.admin.thinkmodelfunction.thingModel.ThingModelRespVO;
import cn.iocoder.yudao.module.iot.dal.dataobject.product.IotProductDO;
import cn.iocoder.yudao.module.iot.dal.dataobject.tdengine.*;
import cn.iocoder.yudao.module.iot.dal.dataobject.tdengine.FieldParser;
import cn.iocoder.yudao.module.iot.dal.dataobject.tdengine.TdFieldDO;
import cn.iocoder.yudao.module.iot.dal.dataobject.tdengine.TdRestApi;
import cn.iocoder.yudao.module.iot.dal.dataobject.thinkmodelfunction.IotThinkModelFunctionDO;
import cn.iocoder.yudao.module.iot.dal.tdengine.TdEngineMapper;
import cn.iocoder.yudao.module.iot.enums.product.IotProductFunctionTypeEnum;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@ -22,147 +25,206 @@ import java.util.stream.Collectors;
public class IotDbStructureDataServiceImpl implements IotDbStructureDataService {
@Resource
private TdEngineMapper tdEngineMapper;
private TdEngineService tdEngineService;
@Resource
private TdRestApi tdRestApi;
@Value("${spring.datasource.dynamic.datasource.tdengine.url}")
private String url;
@Override
public void createSuperTable(ThingModelRespVO thingModel, Integer deviceType) {
// 获取物模型中的属性定义
List<TdField> fields = FieldParser.parse(thingModel);
String tbName = getProductPropertySTableName(deviceType, thingModel.getProductKey());
// 1. 解析物模型，获得字段列表
List<TdFieldDO> schemaFields = new ArrayList<>();
schemaFields.add(TdFieldDO.builder().
fieldName("time").
dataType("TIMESTAMP").
build());
schemaFields.addAll(FieldParser.parse(thingModel));
// 生成创建超级表的 SQL
String sql = TableManager.getCreateSTableSql(tbName, fields, new TdField("device_id", "NCHAR", 64));
if (sql == null) {
log.warn("生成的 SQL 为空，无法创建超级表");
return;
}
log.info("执行 SQL: {}", sql);
// 3. 设置超级表的标签
List<TdFieldDO> tagsFields = new ArrayList<>();
tagsFields.add(TdFieldDO.builder().
fieldName("product_key").
dataType("NCHAR").
dataLength(64).
build());
tagsFields.add(TdFieldDO.builder().
fieldName("device_key").
dataType("NCHAR").
dataLength(64).
build());
tagsFields.add(TdFieldDO.builder().
fieldName("device_name").
dataType("NCHAR").
dataLength(64).
build());
// 年
tagsFields.add(TdFieldDO.builder().
fieldName("year").
dataType("INT").
build());
// 月
tagsFields.add(TdFieldDO.builder().
fieldName("month").
dataType("INT").
build());
// 日
tagsFields.add(TdFieldDO.builder().
fieldName("day").
dataType("INT").
build());
// 时
tagsFields.add(TdFieldDO.builder().
fieldName("hour").
dataType("INT").
build());
// 执行 SQL 创建超级表
tdEngineMapper.createSuperTableDevice(sql);
// 4. 获取超级表的名称
String superTableName = getProductPropertySTableName(deviceType, thingModel.getProductKey());
// 5. 创建超级表
String dataBaseName = url.substring(url.lastIndexOf("/") + 1);
tdEngineService.createSuperTable(schemaFields, tagsFields, dataBaseName, superTableName);
}
@Override
public void updateSuperTable(ThingModelRespVO thingModel, Integer deviceType) {
try {
// 获取旧字段信息
String tbName = getProductPropertySTableName(deviceType, thingModel.getProductKey());
String sql = TableManager.getDescTableSql(tbName);
TdResponse response = tdRestApi.execSql(sql);
if (response.getCode() != TdResponse.CODE_SUCCESS) {
throw new RuntimeException("获取表描述错误: " + JSONUtil.toJsonStr(response));
}
List<TdFieldDO> oldFields = getTableFields(tbName);
List<TdFieldDO> newFields = FieldParser.parse(thingModel);
List<TdField> oldFields = FieldParser.parse(response.getData());
List<TdField> newFields = FieldParser.parse(thingModel);
// 找出新增的字段
List<TdField> addFields = newFields.stream()
.filter(f -> oldFields.stream().noneMatch(old -> old.getName().equals(f.getName())))
.collect(Collectors.toList());
if (!addFields.isEmpty()) {
sql = TableManager.getAddSTableColumnSql(tbName, addFields);
response = tdRestApi.execSql(sql);
if (response.getCode() != TdResponse.CODE_SUCCESS) {
throw new RuntimeException("添加表字段错误: " + JSONUtil.toJsonStr(response));
}
}
// 找出修改的字段
List<TdField> modifyFields = newFields.stream()
.filter(f -> oldFields.stream().anyMatch(old ->
old.getName().equals(f.getName()) &&
(!old.getType().equals(f.getType()) || old.getLength() != f.getLength())))
.collect(Collectors.toList());
if (!modifyFields.isEmpty()) {
sql = TableManager.getModifySTableColumnSql(tbName, modifyFields);
response = tdRestApi.execSql(sql);
if (response.getCode() != TdResponse.CODE_SUCCESS) {
throw new RuntimeException("修改表字段错误: " + JSONUtil.toJsonStr(response));
}
}
// 找出删除的字段
List<TdField> dropFields = oldFields.stream()
.filter(f -> !"time".equals(f.getName()) && !"device_id".equals(f.getName()) &&
newFields.stream().noneMatch(n -> n.getName().equals(f.getName())))
.collect(Collectors.toList());
if (!dropFields.isEmpty()) {
sql = TableManager.getDropSTableColumnSql(tbName, dropFields);
response = tdRestApi.execSql(sql);
if (response.getCode() != TdResponse.CODE_SUCCESS) {
throw new RuntimeException("删除表字段错误: " + JSONUtil.toJsonStr(response));
}
}
updateTableFields(tbName, oldFields, newFields);
} catch (Throwable e) {
log.error("更新物模型超级表失败", e);
}
}
// 获取表字段
private List<TdFieldDO> getTableFields(String tableName) {
List<TdFieldDO> fields = new ArrayList<>();
// 获取超级表的描述信息
List<Map<String, Object>> maps = tdEngineService.describeSuperTable(url.substring(url.lastIndexOf("/") + 1), tableName);
if (maps != null) {
// 过滤掉 note 字段为 TAG 的记录
maps = maps.stream().filter(map -> !"TAG".equals(map.get("note"))).toList();
// 过滤掉 time 字段
maps = maps.stream().filter(map -> !"time".equals(map.get("field"))).toList();
// 解析字段信息
fields = FieldParser.parse(maps.stream()
.map(map -> List.of(map.get("field"), map.get("type"), map.get("length")))
.collect(Collectors.toList()));
}
return fields;
}
// 更新表字段
private void updateTableFields(String tableName, List<TdFieldDO> oldFields, List<TdFieldDO> newFields) {
// 获取新增字段
List<TdFieldDO> addFields = getAddFields(oldFields, newFields);
// 获取修改字段
List<TdFieldDO> modifyFields = getModifyFields(oldFields, newFields);
// 获取删除字段
List<TdFieldDO> dropFields = getDropFields(oldFields, newFields);
String dataBaseName = url.substring(url.lastIndexOf("/") + 1);
// 添加新增字段
if (CollUtil.isNotEmpty(addFields)) {
tdEngineService.addColumnForSuperTable(dataBaseName,tableName, addFields);
}
// 删除旧字段
if (CollUtil.isNotEmpty(dropFields)) {
tdEngineService.dropColumnForSuperTable(dataBaseName,tableName, dropFields);
}
// 修改字段（先删除再添加）
if (CollUtil.isNotEmpty(modifyFields)) {
tdEngineService.dropColumnForSuperTable(dataBaseName,tableName, modifyFields);
tdEngineService.addColumnForSuperTable(dataBaseName,tableName, modifyFields);
}
}
// 获取新增字段
private List<TdFieldDO> getAddFields(List<TdFieldDO> oldFields, List<TdFieldDO> newFields) {
return newFields.stream()
.filter(f -> oldFields.stream().noneMatch(old -> old.getFieldName().equals(f.getFieldName())))
.collect(Collectors.toList());
}
// 获取修改字段
private List<TdFieldDO> getModifyFields(List<TdFieldDO> oldFields, List<TdFieldDO> newFields) {
return newFields.stream()
.filter(f -> oldFields.stream().anyMatch(old ->
old.getFieldName().equals(f.getFieldName()) &&
(!old.getDataType().equals(f.getDataType()) || !Objects.equals(old.getDataLength(), f.getDataLength()))))
.collect(Collectors.toList());
}
// 获取删除字段
private List<TdFieldDO> getDropFields(List<TdFieldDO> oldFields, List<TdFieldDO> newFields) {
return oldFields.stream()
.filter(f -> !"time".equals(f.getFieldName()) && !"device_id".equals(f.getFieldName()) &&
newFields.stream().noneMatch(n -> n.getFieldName().equals(f.getFieldName())))
.collect(Collectors.toList());
}
@Override
public void createSuperTableDataModel(IotProductDO product, List<IotThinkModelFunctionDO> functionList) {
// 1. 生成 ThingModelRespVO
ThingModelRespVO thingModel = new ThingModelRespVO();
thingModel.setId(product.getId());
thingModel.setProductKey(product.getProductKey());
ThingModelRespVO thingModel = buildThingModel(product, functionList);
// 1.1 设置属性、服务和事件
ThingModelRespVO.Model model = new ThingModelRespVO.Model();
List<ThingModelProperty> properties = new ArrayList<>();
// 1.2 遍历功能列表并分类
for (IotThinkModelFunctionDO function : functionList) {
if (Objects.requireNonNull(IotProductFunctionTypeEnum.valueOf(function.getType())) == IotProductFunctionTypeEnum.PROPERTY) {
ThingModelProperty property = new ThingModelProperty();
property.setIdentifier(function.getIdentifier());
property.setName(function.getName());
property.setDescription(function.getDescription());
property.setDataType(function.getProperty().getDataType());
properties.add(property);
}
}
// 1.3 判断属性列表是否为空
if (properties.isEmpty()) {
if (thingModel.getModel().getProperties().isEmpty()) {
log.warn("物模型属性列表为空，不创建超级表");
return;
}
model.setProperties(properties);
thingModel.setModel(model);
String superTableName = getProductPropertySTableName(product.getDeviceType(), product.getProductKey());
String dataBaseName = url.substring(url.lastIndexOf("/") + 1);
Integer tableExists = tdEngineService.checkSuperTableExists(dataBaseName, superTableName);
// 2. 判断是否已经创建,如果已经创建则进行更新
String tbName = getProductPropertySTableName(product.getDeviceType(), product.getProductKey());
Integer iot = tdEngineMapper.checkTableExists("ruoyi_vue_pro", tbName);
if (iot != null && iot > 0) {
// 3. 更新
if (tableExists != null && tableExists > 0) {
updateSuperTable(thingModel, product.getDeviceType());
} else {
// 4. 创建
createSuperTable(thingModel, product.getDeviceType());
}
}
/**
* 根据产品key获取产品属性超级表名
*/
static String getProductPropertySTableName(Integer deviceType, String productKey) {
if (deviceType == 1) {
return String.format("gateway_sub_" + productKey).toLowerCase();
} else if (deviceType == 2) {
return String.format("gateway_" + productKey).toLowerCase();
} else {
return String.format("device_" + productKey).toLowerCase();
}
private ThingModelRespVO buildThingModel(IotProductDO product, List<IotThinkModelFunctionDO> functionList) {
ThingModelRespVO thingModel = new ThingModelRespVO();
thingModel.setId(product.getId());
thingModel.setProductKey(product.getProductKey());
ThingModelRespVO.Model model = new ThingModelRespVO.Model();
List<ThingModelProperty> properties = functionList.stream()
.filter(function -> IotProductFunctionTypeEnum.PROPERTY.equals(IotProductFunctionTypeEnum.valueOf(function.getType())))
.map(this::buildThingModelProperty)
.collect(Collectors.toList());
model.setProperties(properties);
thingModel.setModel(model);
return thingModel;
}
/**
* 根据deviceId获取设备属性表名
*/
static String getDevicePropertyTableName(String deviceType, String productKey, String deviceKey) {
return String.format(deviceType + "_" + productKey + "_" + deviceKey).toLowerCase();
private ThingModelProperty buildThingModelProperty(IotThinkModelFunctionDO function) {
ThingModelProperty property = new ThingModelProperty();
property.setIdentifier(function.getIdentifier());
property.setName(function.getName());
property.setDescription(function.getDescription());
property.setDataType(function.getProperty().getDataType());
return property;
}
}
static String getProductPropertySTableName(Integer deviceType, String productKey) {
return switch (deviceType) {
case 1 -> String.format("gateway_sub_%s", productKey).toLowerCase();
case 2 -> String.format("gateway_%s", productKey).toLowerCase();
default -> String.format("device_%s", productKey).toLowerCase();
};
}
static String getDevicePropertyTableName(String deviceType, String productKey, String deviceKey) {
return String.format("%s_%s_%s", deviceType, productKey, deviceKey).toLowerCase();
}
}

75
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/service/tdengine/TdEngineService.java
View File

@ -1,6 +1,6 @@
package cn.iocoder.yudao.module.iot.service.tdengine;
import cn.iocoder.yudao.module.iot.domain.FieldsVo;
import cn.iocoder.yudao.module.iot.dal.dataobject.tdengine.TdFieldDO;
import cn.iocoder.yudao.module.iot.domain.SelectDto;
import cn.iocoder.yudao.module.iot.domain.TableDto;
import cn.iocoder.yudao.module.iot.domain.TagsSelectDao;
@ -27,12 +27,44 @@ public interface TdEngineService {
*
* @param schemaFields schema字段
* @param tagsFields tags字段
* @param dataBaseName 数据库名称
* @param superTableName 超级表名称
* @throws Exception 异常
*/
void createSuperTable(List<FieldsVo> schemaFields, List<FieldsVo> tagsFields, String dataBaseName,
String superTableName) throws Exception;
void createSuperTable(List<TdFieldDO> schemaFields, List<TdFieldDO> tagsFields, String dataBaseName, String superTableName);
/**
* 检查超级表是否存在
*/
Integer checkSuperTableExists(String dataBaseName, String superTableName);
/**
* 获取超级表的结构信息
*/
List<Map<String, Object>> describeSuperTable(String dataBaseName, String superTableName);
/**
* 为超级表添加列
*
* @param dataBaseName 数据库名称
* @param superTableName 超级表名称
* @param fieldsVo 字段信息
*/
void addColumnForSuperTable(String dataBaseName,String superTableName, List<TdFieldDO> fieldsVo);
/**
* 为超级表删除列
*
* @param dataBaseName 数据库名称
* @param superTableName 超级表名称
* @param fieldsVo 字段信息
*/
void dropColumnForSuperTable(String dataBaseName,String superTableName, List<TdFieldDO> fieldsVo);
/**
* 为超级表添加tag
*/
Long getCountByTimesTamp(SelectDto selectDto) throws Exception;
/**
* 创建表
@ -59,37 +91,6 @@ public interface TdEngineService {
*/
List<Map<String, Object>> selectByTimesTamp(SelectDto selectDto) throws Exception;
/**
* 为超级表添加列
*
* @param superTableName 超级表名称
* @param fieldsVo 字段信息
* @throws Exception 异常
*/
void addColumnForSuperTable(String superTableName, FieldsVo fieldsVo) throws Exception;
/**
* 为超级表删除列
*
* @param superTableName 超级表名称
* @param fieldsVo 字段信息
* @throws Exception 异常
*/
void dropColumnForSuperTable(String superTableName, FieldsVo fieldsVo) throws Exception;
/**
* 为超级表添加tag
*/
Long getCountByTimesTamp(SelectDto selectDto) throws Exception;
/**
* 检查表是否存在
*
* @return 1存在 0不存在
* @throws Exception 异常
*/
Integer checkTableExists(SelectDto selectDto) throws Exception;
/**
* 初始化超级表
*
@ -138,4 +139,6 @@ public interface TdEngineService {
* @return 数据
*/
List<Map<String, Object>> getAggregateData(SelectVisualDto selectVisualDto);
}

44
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/service/tdengine/TdEngineServiceImpl.java
View File

@ -1,10 +1,12 @@
package cn.iocoder.yudao.module.iot.service.tdengine;
import cn.iocoder.yudao.module.iot.domain.FieldsVo;
import cn.iocoder.yudao.module.iot.dal.dataobject.tdengine.TdFieldDO;
import cn.iocoder.yudao.module.iot.dal.tdengine.TdEngineMapper;
import cn.iocoder.yudao.module.iot.domain.SelectDto;
import cn.iocoder.yudao.module.iot.domain.TableDto;
import cn.iocoder.yudao.module.iot.domain.TagsSelectDao;
import cn.iocoder.yudao.module.iot.domain.visual.SelectVisualDto;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@ -15,25 +17,27 @@ import java.util.Map;
@Slf4j
public class TdEngineServiceImpl implements TdEngineService {
@Resource
private TdEngineMapper tdEngineMapper;
@Override
public void createDateBase(String dataBaseName) {
tdEngineMapper.createDatabase(dataBaseName);
}
@Override
public void createSuperTable(List<FieldsVo> schemaFields, List<FieldsVo> tagsFields, String dataBaseName, String superTableName) {
public void createSuperTable(List<TdFieldDO> schemaFields, List<TdFieldDO> tagsFields, String dataBaseName, String superTableName) {
tdEngineMapper.createSuperTable(schemaFields, tagsFields, dataBaseName, superTableName);
}
@Override
public void createTable(TableDto tableDto) {
tdEngineMapper.createTable(tableDto);
}
@Override
public void insertData(TableDto tableDto) {
tdEngineMapper.insertData(tableDto);
}
@Override
@ -42,13 +46,17 @@ public class TdEngineServiceImpl implements TdEngineService {
}
@Override
public void addColumnForSuperTable(String superTableName, FieldsVo fieldsVo) {
public void addColumnForSuperTable(String dataBaseName,String superTableName, List<TdFieldDO> fields) {
for (TdFieldDO field : fields) {
tdEngineMapper.addColumnForSuperTable(dataBaseName,superTableName, field);
}
}
@Override
public void dropColumnForSuperTable(String superTableName, FieldsVo fieldsVo) {
public void dropColumnForSuperTable(String dataBaseName,String superTableName, List<TdFieldDO> fields) {
for (TdFieldDO field : fields) {
tdEngineMapper.dropColumnForSuperTable(dataBaseName,superTableName, field);
}
}
@Override
@ -56,11 +64,6 @@ public class TdEngineServiceImpl implements TdEngineService {
return 0L;
}
@Override
public Integer checkTableExists(SelectDto selectDto) {
return 0;
}
@Override
public void initSTableFrame(String msg) {
@ -90,4 +93,15 @@ public class TdEngineServiceImpl implements TdEngineService {
public List<Map<String, Object>> getAggregateData(SelectVisualDto selectVisualDto) {
return List.of();
}
@Override
public Integer checkSuperTableExists(String dataBaseName, String superTableName) {
List<Map<String, Object>> results = tdEngineMapper.showSuperTables(dataBaseName, superTableName);
return results == null || results.isEmpty() ? 0 : results.size();
}
@Override
public List<Map<String, Object>> describeSuperTable(String dataBaseName, String superTableName) {
return tdEngineMapper.describeSuperTable(dataBaseName, superTableName);
}
}

2
yudao-module-iot/yudao-module-iot-biz/src/main/java/cn/iocoder/yudao/module/iot/service/thinkmodelfunction/IotThinkModelFunctionServiceImpl.java
View File

@ -174,8 +174,10 @@ public class IotThinkModelFunctionServiceImpl implements IotThinkModelFunctionSe
public void createSuperTableDataModel(Long productId) {
// 1. 查询产品
IotProductDO product = productService.getProduct(productId);
// 2. 查询产品的物模型功能列表
List<IotThinkModelFunctionDO> functionList = thinkModelFunctionMapper.selectListByProductId(productId);
// 3. 生成 TDengine 的数据模型
dbStructureDataService.createSuperTableDataModel(product, functionList);
}

223
yudao-module-iot/yudao-module-iot-biz/src/main/resources/mapper/tdengine/TdEngineMapper.xml
View File

@ -8,7 +8,7 @@
</update>
<update id="createSuperTable">
create table if not exists #{dataBaseName}.#{superTableName}
CREATE STABLE IF NOT EXISTS ${dataBaseName}.${superTableName}
<foreach item="item" collection="schemaFields" separator=","
open="(" close=")" index="">
<if test="item.fieldName != null || item.fieldName != ''">
@ -16,46 +16,48 @@
</if>
<if test="item.dataType != null || item.dataType != ''">
<choose>
<when test="item.dataType == 'timestamp'">
timestamp
<when test="item.dataType == 'TIMESTAMP'">
TIMESTAMP
</when>
<when test="item.dataType == 'tinyint'">
tinyint
<when test="item.dataType == 'TINYINT'">
TINYINT
</when>
<when test="item.dataType == 'smallint'">
smallint
<when test="item.dataType == 'SMALLINT'">
SMALLINT
</when>
<when test="item.dataType == 'int'">
int
<when test="item.dataType == 'INT'">
INT
</when>
<when test="item.dataType == 'bigint'">
bigint
<when test="item.dataType == 'BIGINT'">
BIGINT
</when>
<when test="item.dataType == 'float'">
float
<when test="item.dataType == 'FLOAT'">
FLOAT
</when>
<when test="item.dataType == 'double'">
double
<when test="item.dataType == 'DOUBLE'">
DOUBLE
</when>
<when test="item.dataType == 'binary'">
binary
<when test="item.dataType == 'BINARY'">
BINARY
</when>
<when test="item.dataType == 'nchar'">
nchar
<when test="item.dataType == 'NCHAR'">
NCHAR
</when>
<when test="item.dataType == 'bool'">
bool
<when test="item.dataType == 'BOOL'">
BOOL
</when>
<when test="item.dataType == 'json'">
json
<when test="item.dataType == 'JSON'">
JSON
</when>
</choose>
</if>
<if test="item.size != null">
(#{item.size})
<if test="item.dataLength > 0">
(
${item.dataLength}
)
</if>
</foreach>
tags
TAGS
<!--tdEngine不支持动态tags里的数据类型，只能使用choose标签比对-->
<foreach item="item" collection="tagsFields" separator=","
open="(" close=")" index="">
@ -64,43 +66,45 @@
</if>
<if test="item.dataType != null || item.dataType != ''">
<choose>
<when test="item.dataType == 'timestamp'">
timestamp
<when test="item.dataType == 'TIMESTAMP'">
TIMESTAMP
</when>
<when test="item.dataType == 'tinyint'">
tinyint
<when test="item.dataType == 'TINYINT'">
TINYINT
</when>
<when test="item.dataType == 'smallint'">
smallint
<when test="item.dataType == 'SMALLINT'">
SMALLINT
</when>
<when test="item.dataType == 'int'">
int
<when test="item.dataType == 'INT'">
INT
</when>
<when test="item.dataType == 'bigint'">
bigint
<when test="item.dataType == 'BIGINT'">
BIGINT
</when>
<when test="item.dataType == 'float'">
float
<when test="item.dataType == 'FLOAT'">
FLOAT
</when>
<when test="item.dataType == 'double'">
double
<when test="item.dataType == 'DOUBLE'">
DOUBLE
</when>
<when test="item.dataType == 'binary'">
binary
<when test="item.dataType == 'BINARY'">
BINARY
</when>
<when test="item.dataType == 'nchar'">
nchar
<when test="item.dataType == 'NCHAR'">
NCHAR
</when>
<when test="item.dataType == 'bool'">
bool
<when test="item.dataType == 'BOOL'">
BOOL
</when>
<when test="item.dataType == 'json'">
json
<when test="item.dataType == 'JSON'">
JSON
</when>
</choose>
</if>
<if test="item.size != null">
(#{item.size})
<if test="item.dataLength > 0">
(
${item.dataLength}
)
</if>
</foreach>
</update>
@ -135,7 +139,6 @@
</foreach>
</insert>
<select id="selectByTimestamp" parameterType="cn.iocoder.yudao.module.iot.domain.SelectDto"
resultType="Map">
select * from #{dataBaseName}.#{tableName}
@ -146,66 +149,58 @@
</select>
<update id="addColumnForSuperTable">
ALTER
STABLE
#{superTableName}
ADD
COLUMN
<if test="fieldsVo.fieldName != null || fieldsVo.fieldName != ''">
#{fieldsVo.fieldName}
ALTER STABLE ${dataBaseName}.${superTableName} ADD COLUMN
<if test="field.fieldName != null || field.fieldName != ''">
#{field.fieldName}
</if>
<if test="fieldsVo.dataType != null || fieldsVo.dataType != ''">
<if test="field.dataType != null || field.dataType != ''">
<choose>
<when test="fieldsVo.dataType == 'timestamp'">
timestamp
<when test="field.dataType == 'TIMESTAMP'">
TIMESTAMP
</when>
<when test="fieldsVo.dataType == 'tinyint'">
tinyint
<when test="field.dataType == 'TINYINT'">
TINYINT
</when>
<when test="fieldsVo.dataType == 'smallint'">
smallint
<when test="field.dataType == 'SMALLINT'">
SMALLINT
</when>
<when test="fieldsVo.dataType == 'int'">
int
<when test="field.dataType == 'INT'">
INT
</when>
<when test="fieldsVo.dataType == 'bigint'">
bigint
<when test="field.dataType == 'BIGINT'">
BIGINT
</when>
<when test="fieldsVo.dataType == 'float'">
float
<when test="field.dataType == 'FLOAT'">
FLOAT
</when>
<when test="fieldsVo.dataType == 'double'">
double
<when test="field.dataType == 'DOUBLE'">
DOUBLE
</when>
<when test="fieldsVo.dataType == 'binary'">
binary
<when test="field.dataType == 'BINARY'">
BINARY
</when>
<when test="fieldsVo.dataType == 'nchar'">
nchar
<when test="field.dataType == 'NCHAR'">
NCHAR
</when>
<when test="fieldsVo.dataType == 'bool'">
bool
<when test="field.dataType == 'BOOL'">
BOOL
</when>
<when test="fieldsVo.dataType == 'json'">
json
<when test="field.dataType == 'JSON'">
JSON
</when>
</choose>
</if>
<if test="fieldsVo.size != null">
<if test="field.dataLength > 0">
(
#{fieldsVo.size}
#{field.dataLength}
)
</if>
</update>
<update id="dropColumnForSuperTable">
ALTER
STABLE
#{superTableName}
DROP
COLUMN
<if test="fieldsVo.fieldName != null || fieldsVo.fieldName != ''">
#{fieldsVo.fieldName}
ALTER STABLE ${dataBaseName}.${superTableName} DROP COLUMN
<if test="field.fieldName != null || field.fieldName != ''">
#{field.fieldName}
</if>
</update>
@ -215,49 +210,49 @@
#{superTableName}
ADD
TAG
<if test="fieldsVo.fieldName != null || fieldsVo.fieldName != ''">
#{fieldsVo.fieldName}
<if test="field.fieldName != null || fieldDO.fieldName != ''">
#{fieldDO.fieldName}
</if>
<if test="fieldsVo.dataType != null || fieldsVo.dataType != ''">
<if test="fieldDO.dataType != null || fieldDO.dataType != ''">
<choose>
<when test="fieldsVo.dataType == 'timestamp'">
<when test="fieldDO.dataType == 'timestamp'">
timestamp
</when>
<when test="fieldsVo.dataType == 'tinyint'">
<when test="fieldDO.dataType == 'tinyint'">
tinyint
</when>
<when test="fieldsVo.dataType == 'smallint'">
<when test="fieldDO.dataType == 'smallint'">
smallint
</when>
<when test="fieldsVo.dataType == 'int'">
<when test="fieldDO.dataType == 'int'">
int
</when>
<when test="fieldsVo.dataType == 'bigint'">
<when test="fieldDO.dataType == 'bigint'">
bigint
</when>
<when test="fieldsVo.dataType == 'float'">
<when test="fieldDO.dataType == 'float'">
float
</when>
<when test="fieldsVo.dataType == 'double'">
<when test="fieldDO.dataType == 'double'">
double
</when>
<when test="fieldsVo.dataType == 'binary'">
<when test="fieldDO.dataType == 'binary'">
binary
</when>
<when test="fieldsVo.dataType == 'nchar'">
<when test="fieldDO.dataType == 'nchar'">
nchar
</when>
<when test="fieldsVo.dataType == 'bool'">
<when test="fieldDO.dataType == 'bool'">
bool
</when>
<when test="fieldsVo.dataType == 'json'">
<when test="fieldDO.dataType == 'json'">
json
</when>
</choose>
</if>
<if test="fieldsVo.size != null">
<if test="fieldDO.dataLength > 0">
(
#{fieldsVo.size}
#{fieldDO.dataLength}
)
</if>
</update>
@ -279,11 +274,8 @@
FROM #{dataBaseName}.#{tableName} WHERE ${fieldName} BETWEEN #{startTime} AND #{endTime}
</select>
<select id="checkTableExists" resultType="java.lang.Integer">
SELECT COUNT(0)
FROM information_schema.ins_tables
WHERE db_name = #{dataBaseName}
AND table_name = #{tableName}
<select id="showSuperTables" resultType="java.util.Map">
SHOW ${dataBaseName}.STABLES LIKE '${superTableName}'
</select>
<select id="getLastData" resultType="java.util.Map">
@ -304,11 +296,13 @@
FROM #{dataBaseName}.#{tableName} WHERE ts BETWEEN #{startTime} AND #{endTime}
LIMIT #{num}
</select>
<select id="getRealtimeData" resultType="java.util.Map"
parameterType="cn.iocoder.yudao.module.iot.domain.visual.SelectVisualDto">
SELECT #{fieldName}, ts
FROM #{dataBaseName}.#{tableName} LIMIT #{num}
</select>
<select id="getAggregateData" resultType="java.util.Map"
parameterType="cn.iocoder.yudao.module.iot.domain.visual.SelectVisualDto">
SELECT #{aggregate}(${fieldName})
@ -316,9 +310,8 @@
LIMIT #{num}
</select>
<insert id="createSuperTableDevice">
${sql}
</insert>
<select id="describeSuperTable" resultType="java.util.Map">
DESCRIBE ${dataBaseName}.${superTableName}
</select>
</mapper>