csv 文件就是文本文件,用 txtreader 和 txtwriter 读写。配置文件详细语法请参见 。

txtreader配置示例:

txtwriter 配置示例:

  1. "name":"txtfilewriter",
  2. "parameter":{
  3. "path":"文件全路径",
  4. "fileName":"文件名",
  5. "writeMode":"truncate",
  6. "dateFormat":"yyyy-MM-dd",
  7. "charset":"UTF-8",
  8. "nullFormat":"",
  9. "fileDelimiter":"||"
  10. }
  11. }

针对 MySQL 数据库,用 mysqlreader 和 mysqlwriter 插件读写。

mysqlreader 配置示例:

  1. "reader": {
  2. "name": "mysqlreader",
  3. "parameter": {
  4. "username": "root",
  5. "password": "root",
  6. "column": [
  7. "id",
  8. "name"
  9. ],
  10. "splitPk": "db_id",
  11. "connection": [
  12. {
  13. "table": [
  14. "table"
  15. ],
  16. "jdbcUrl": [
  17. "jdbc:mysql://127.0.0.1:3306/database"
  18. ]
  19. }
  20. ]
  21. }
  22. }

针对 Oracle 数据库,用 oraclereader 和 oraclewriter 插件来读写。

oraclereader 配置示例:

  1. "reader": {
  2. "name": "oraclereader",
  3. "parameter": {
  4. // 数据库连接用户名
  5. "username": "root",
  6. // 数据库连接密码
  7. "password": "root",
  8. "column": [
  9. "id","name"
  10. ],
  11. //切分主键
  12. "splitPk": "db_id",
  13. "connection": [
  14. {
  15. "table": [
  16. "table"
  17. ],
  18. "jdbcUrl": [
  19. "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]"
  20. ]
  21. }
  22. ]
  23. }
  24. }

oraclewriter 配置示例:

  1. "writer": {
  2. "name": "oraclewriter",
  3. "parameter": {
  4. "username": "root",
  5. "password": "root",
  6. "column": [
  7. "id",
  8. "name"
  9. ],
  10. "preSql": [
  11. ],
  12. "connection": [
  13. "jdbcUrl": "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]",
  14. "table": [
  15. "test"
  16. ]
  17. }
  18. ]
  19. }
  20. }

db2reader 配置示例:

db2writer 配置示例:

  • oceanbasev10reader 配置示例
  1. "reader":{
  2. "name":"oceanbasev10reader",
  3. "parameter":{
  4. "where":"",
  5. "timeout":10000,
  6. "readBatchSize":100000,
  7. "readByPartition":"true",
  8. "column": [
  9. “列名1”,”列名2
  10. ],
  11. "connection":[
  12. {
  13. "jdbcUrl":["||_dsc_ob10_dsc_||集群名:租户名||_dsc_ob10_dsc_||jdbc:oceanbase://连接IP:连接端口/模式名或数据库名"],
  14. "table":["表名"]
  15. }
  16. ],
  17. "username":"租户内用户名",
  18. "password":"密码"
  19. }
  20. }

示例:OceanBase 表 ware 导出到 csv 文件

  1. [admin@*** /home/admin/datax3]
  2. $cat job/ob_tpcc_ware_2_csv.json
  3. {
  4. "job":{
  5. "setting":{
  6. "speed":{
  7. "channel":10
  8. },
  9. "errorLimit":{
  10. "record":0, "percentage": 0.02
  11. }
  12. },
  13. "content":[
  14. {
  15. "reader":{
  16. "name":"oceanbasev10reader",
  17. "parameter":{
  18. "where":"",
  19. "timeout":10000,
  20. "readBatchSize":100000,
  21. "readByPartition":"true",
  22. "column": [
  23. "W_ID","W_YTD","W_TAX","W_NAME","W_STREET_1","W_STREET_2","W_CITY","W_STATE","W_ZIP"
  24. ],
  25. "connection":[
  26. {
  27. "jdbcUrl":["||_dsc_ob10_dsc_||obdemo:obbmsql||_dsc_ob10_dsc_||jdbc:oceanbase://127.1:2883/tpcc"],
  28. "table":["ware"]
  29. }
  30. ],
  31. "username":"tpcc",
  32. "password":"123456"
  33. }
  34. },
  35. "writer":{
  36. "name":"txtfilewriter",
  37. "parameter":{
  38. "path":"/home/admin/csvdata/",
  39. "fileName":"ware",
  40. "writeMode":"truncate",
  41. "charset":"UTF-8",
  42. "fileDelimiter":"||"
  43. }
  44. }
  45. }
  46. ]
  47. }
  48. }
  49. [admin@*** /home/admin/datax3]
  50. $bin/datax.py job/ob_tpcc_ware_2_csv.json

  • oceanbasev10writer**配置示例**

使用 DataX 向 OceanBase 里写入时,要避免写入速度过快导致 OceanBase 的增量内存耗尽。通常建议 DataX 配置文件里针对写入做一个写入限速设置。关键字是 memstoreThreshold :

示例:从 csv 文件导入到 OceanBase 表中

  1. [admin@*** /home/admin/datax3]
  2. $cat job/csv_2_ob_tpcc_ware2.json
  3. {
  4. "job":{
  5. "setting":{
  6. "speed":{
  7. "channel":32
  8. },
  9. "errorLimit":{
  10. "record":0, "percentage": 0.02
  11. }
  12. },
  13. "content":[
  14. {
  15. "reader":{
  16. "name":"txtfilereader",
  17. "parameter":{
  18. "path":["/home/admin/csvdata/ware*"],
  19. "encoding":"UTF-8",
  20. "column":[
  21. { "index":0, "type":"long" }
  22. ,{ "index":1, "type":"long" }
  23. ,{ "index":2, "type":"long" }
  24. ,{ "index":3, "type":"string" }
  25. ,{ "index":4, "type":"string" }
  26. ,{ "index":5, "type":"string" }
  27. ,{ "index":6, "type":"string" }
  28. ,{ "index":7, "type":"string" }
  29. ,{ "index":8, "type":"string" }
  30. ],
  31. "fieldDelimiter":",",
  32. "fileFormat":"text"
  33. }
  34. },
  35. "writer":{
  36. "name":"oceanbasev10writer",
  37. "parameter":{
  38. "writeMode":"insert",
  39. "column":[
  40. "W_ID","W_YTD","W_TAX","W_NAME","W_STREET_1","W_STREET_2","W_CITY","W_STATE","W_ZIP"
  41. ],
  42. "connection":[
  43. {
  44. "jdbcUrl":"||_dsc_ob10_dsc_||obdemo:obbmsql||_dsc_ob10_dsc_||jdbc:oceanbase://127.1:2883/tpcc",
  45. "table":["WARE2"]
  46. }
  47. ],
  48. "username":"tpcc",
  49. "password":"123456",
  50. "batchSize":256,
  51. " memstoreThreshold":"90"
  52. }
  53. }
  54. }
  55. ]
  56. }
  57. }

image.png