安装部署

    有了上述编译阶段产出的tar.gz包后，解压后就可以启动inlong-sort-standalone的应用了。
    示例：

    • 可以从ClassPath的SortClusterConfig.conf源文件读取，但不支持实时更新
    • 可以从Inlong Manager的HTTP接口获取配置
      | 配置名 | 是否必须 | 默认值 |描述 | | —————— | —————— | —————— | —————— | |clusterName | Y | NA | 用来唯一标识一个inlong-sort-standalone集群 | |sortTasks | Y | NA | 分发任务列表 |
    配置名是否必须默认值描述
    nameYNA分发任务名
    typeYNA分发任务类型，如HIVE(“hive”), TUBE(“tube”), KAFKA(“kafka”), PULSAR(“pulsar”), ElasticSearch(“ElasticSearch”), UNKNOWN(“n”)
    idParamsYNAInlong数据流参数列表
    sinkParamsYNA分发任务的参数

    Hive分发任务的idParams

    配置名是否必须默认值描述
    hdfsPathYNAHDFS的NameNode
    maxFileOpenDelayMinuteN5单个HDFS文件最大写入时间，单位分钟
    tokenOvertimeMinuteN60单个Inlong数据流的分区创建token最大占用时间，单位分钟
    maxOutputFileSizeGbN2单个HDFS文件最大大小，单位GB
    hiveJdbcUrlYNAHive的JDBC路径
    hiveDatabaseYNAHive的数据库
    hiveUsernameYNAHive的用户名
    hivePasswordYNAHive的密码

    Pulsar分发任务的idParams

    配置名是否必须默认值描述
    serviceUrlYNAPulsar服务路径
    authenticationYNAPulsar集群鉴权
    enableBatchingNtrueenableBatching
    batchingMaxBytesN5242880batchingMaxBytes
    batchingMaxMessagesN3000batchingMaxMessages
    batchingMaxPublishDelayN1batchingMaxPublishDelay
    maxPendingMessagesN1000maxPendingMessages
    maxPendingMessagesAcrossPartitionsN50000maxPendingMessagesAcrossPartitions
    sendTimeoutN0sendTimeout
    compressionTypeNNONEcompressionType
    blockIfQueueFullNtrueblockIfQueueFull
    roundRobinRouterBatchingPartitionSwitchFrequencyN10roundRobinRouterBatchingPartitionSwitchFrequency

    Hive配置样例

    1. {
    2.     "data":{
    3.         "clusterName":"hivev3-sz-sz1",
    4.         "sortTasks":[
    5.             {
    6.                 "idParams":[
    7.                     {
    8.                         "inlongGroupId":"0fc00000046",
    9.                         "inlongStreamId":"",
    10.                         "separator":"|",
    11.                         "partitionIntervalMs":3600000,
    12.                         "idRootPath":"/user/hive/warehouse/t_inlong_v1_0fc00000046",
    13.                         "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
    14.                         "hiveTableName":"t_inlong_v1_0fc00000046",
    15.                         "partitionFieldName":"dt",
    16.                         "partitionFieldPattern":"yyyyMMddHH",
    17.                         "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
    18.                         "maxPartitionOpenDelayHour":8
    19.                     },
    20.                     {
    21.                         "inlongGroupId":"03600000045",
    22.                         "separator":"|",
    23.                         "partitionIntervalMs":3600000,
    24.                         "idRootPath":"/user/hive/warehouse/t_inlong_v1_03600000045",
    25.                         "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
    26.                         "hiveTableName":"t_inlong_v1_03600000045",
    27.                         "partitionFieldName":"dt",
    28.                         "partitionFieldPattern":"yyyyMMddHH",
    30.                         "maxPartitionOpenDelayHour":8
    31.                     },
    32.                     {
    33.                         "inlongGroupId":"05100054990",
    34.                         "inlongStreamId":"",
    35.                         "separator":"|",
    36.                         "partitionIntervalMs":3600000,
    37.                         "idRootPath":"/user/hive/warehouse/t_inlong_v1_05100054990",
    38.                         "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
    39.                         "hiveTableName":"t_inlong_v1_05100054990",
    40.                         "partitionFieldName":"dt",
    41.                         "partitionFieldPattern":"yyyyMMddHH",
    42.                         "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
    43.                         "maxPartitionOpenDelayHour":8
    44.                     },
    45.                     {
    46.                         "inlongGroupId":"09c00014434",
    47.                         "inlongStreamId":"",
    48.                         "separator":"|",
    49.                         "partitionIntervalMs":3600000,
    50.                         "idRootPath":"/user/hive/warehouse/t_inlong_v1_09c00014434",
    51.                         "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
    52.                         "hiveTableName":"t_inlong_v1_09c00014434",
    53.                         "partitionFieldName":"dt",
    54.                         "partitionFieldPattern":"yyyyMMddHH",
    55.                         "maxPartitionOpenDelayHour":8
    56.                     },
    57.                     {
    59.                         "inlongStreamId":"",
    60.                         "separator":"|",
    61.                         "partitionIntervalMs":3600000,
    62.                         "idRootPath":"/user/hive/warehouse/t_inlong_v1_0c900035509",
    63.                         "partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
    64.                         "hiveTableName":"t_inlong_v1_0c900035509",
    65.                         "partitionFieldName":"dt",
    66.                         "partitionFieldPattern":"yyyyMMddHH",
    67.                         "msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
    68.                         "maxPartitionOpenDelayHour":8
    69.                     }
    70.                 ],
    71.                 "name":"sid_hive_inlong6th_v3",
    72.                 "sinkParams":{
    73.                     "hdfsPath":"hdfs://127.0.0.1:9000",
    74.                     "maxFileOpenDelayMinute":"5",
    75.                     "tokenOvertimeMinute":"60",
    76.                     "maxOutputFileSizeGb":"2",
    77.                     "hiveJdbcUrl":"jdbc:hive2://127.0.0.2:10000",
    78.                     "hiveDatabase":"default",
    79.                     "hiveUsername":"hive",
    80.                     "hivePassword":"hive"
    81.                 },
    82.                 "type":"HIVE"
    83.             }
    84.         ]
    85.     },
    86.     "errCode":0,
    87.     "md5":"md5",
    88. }