Phoenix 进行hive外表映射bug问题研究
Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: Failed with exception Chunk [idcard_app] is not a valid entryjava.lang.IllegalArgumentException: Chunk [idcard_app] is not a valid entry at com.google.common.base.Preconditions.checkArgument(Preconditions.java:115) at com.google.common.base.Splitter$MapSplitter.split(Splitter.java:452) at org.apache.phoenix.hive.util.ColumnMappingUtils.getColumnMappingMap(ColumnMappingUtils.java:51) at org.apache.phoenix.hive.util.ColumnMappingUtils.getReverseColumnMapping(ColumnMappingUtils.java:62) at org.apache.phoenix.hive.mapreduce.PhoenixResultWritable.setConf(PhoenixResultWritable.java:203) at org.apache.phoenix.hive.mapreduce.PhoenixResultWritable.<init>(PhoenixResultWritable.java:69) at org.apache.phoenix.hive.mapreduce.PhoenixResultWritable.<init>(PhoenixResultWritable.java:74) at org.apache.phoenix.hive.PhoenixSerializer.<init>(PhoenixSerializer.java:80) at org.apache.phoenix.hive.PhoenixSerDe.initialize(PhoenixSerDe.java:86) at org.apache.hadoop.hive.serde2.AbstractSerDe.initialize(AbstractSerDe.java:57) at org.apache.hadoop.hive.serde2.SerDeUtils.initializeSerDeWithoutErrorCheck(SerDeUtils.java:542) at org.apache.hadoop.hive.ql.plan.TableDesc.getDeserializer(TableDesc.java:96) at org.apache.hadoop.hive.ql.exec.FetchOperator.setupOutputObjectInspector(FetchOperator.java:564) at org.apache.hadoop.hive.ql.exec.FetchOperator.initialize(FetchOperator.java:172) at org.apache.hadoop.hive.ql.exec.FetchOperator.<init>(FetchOperator.java:140) at org.apache.hadoop.hive.ql.exec.FetchTask.initialize(FetchTask.java:80) at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:592) at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1356) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1343) at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:185) at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:271) at org.apache.hive.service.cli.operation.Operation.run(Operation.java:337) at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:439) at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:416) at sun.reflect.GeneratedMethodAccessor44.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) at com.sun.proxy.$Proxy22.executeStatementAsync(Unknown Source) at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:282) at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:501) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1313) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1298) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
实验:建立一个165个字段的Phoenix,hive进行映射Phoenix表出现上面的报错
CREATE TABLE "FIN_RISK"."T_TONGDUN_PERLOAN_VARIABLE2" (
a1 varchar,
a2 varchar,
a3 varchar,
a4 varchar,
a5 varchar,
a6 varchar,
a7 varchar,
a8 varchar,
a9 varchar,
a10 varchar,
a11 varchar,
a12 varchar,
a13 varchar,
a14 varchar,
a15 varchar,
a16 varchar,
a17 varchar,
a18 varchar,
a19 varchar,
a20 varchar,
a21 varchar,
a22 varchar,
a23 varchar,
a24 varchar,
a25 varchar,
a26 varchar,
a27 varchar,
a28 varchar,
a29 varchar,
a30 varchar,
a31 varchar,
a32 varchar,
a33 varchar,
a34 varchar,
a35 varchar,
a36 varchar,
a37 varchar,
a38 varchar,
a39 varchar,
a40 varchar,
a41 varchar,
a42 varchar,
a43 varchar,
a44 varchar,
a45 varchar,
a46 varchar,
a47 varchar,
a48 varchar,
a49 varchar,
a50 varchar,
a51 varchar,
a52 varchar,
a53 varchar,
a54 varchar,
a55 varchar,
a56 varchar,
a57 varchar,
a58 varchar,
a59 varchar,
a60 varchar,
a61 varchar,
a62 varchar,
a63 varchar,
a64 varchar,
a65 varchar,
a66 varchar,
a67 varchar,
a68 varchar,
a69 varchar,
a70 varchar,
a71 varchar,
a72 varchar,
a73 varchar,
a74 varchar,
a75 varchar,
a76 varchar,
a77 varchar,
a78 varchar,
a79 varchar,
a80 varchar,
a81 varchar,
a82 varchar,
a83 varchar,
a84 varchar,
a85 varchar,
a86 varchar,
a87 varchar,
a88 varchar,
a89 varchar,
a90 varchar,
a91 varchar,
a92 varchar,
a93 varchar,
a94 varchar,
a95 varchar,
a96 varchar,
a97 varchar,
a98 varchar,
a99 varchar,
a100 varchar,
a101 varchar,
a102 varchar,
a103 varchar,
a104 varchar,
a105 varchar,
a106 varchar,
a107 varchar,
a108 varchar,
a109 varchar,
a110 varchar,
a111 varchar,
a112 varchar,
a113 varchar,
a114 varchar,
a115 varchar,
a116 varchar,
a117 varchar,
a118 varchar,
a119 varchar,
a120 varchar,
a121 varchar,
a122 varchar,
a123 varchar,
a124 varchar,
a125 varchar,
a126 varchar,
a127 varchar,
a128 varchar,
a129 varchar,
a130 varchar,
a131 varchar,
a132 varchar,
a133 varchar,
a134 varchar,
a135 varchar,
a136 varchar,
a137 varchar,
a138 varchar,
a139 varchar,
a140 varchar,
a141 varchar,
a142 varchar,
a143 varchar,
a144 varchar,
a145 varchar,
a146 varchar,
a147 varchar,
a148 varchar,
a149 varchar,
a150 varchar,
a151 varchar,
a152 varchar,
a153 varchar,
a154 varchar,
a155 varchar,
a156 varchar,
a157 varchar,
a158 varchar,
a159 varchar,
a160 varchar,
a161 varchar,
a162 varchar,
a163 varchar,
a164 varchar,
a165 varchar,
a166 varchar,
a167 varchar,
a168 varchar,
a169 timestamp
CONSTRAINT PK PRIMARY KEY(a1)
)SALT_BUCKETS=20;
create external table fin_risk.bdl_fin_risk_t_tongdun_perloan_variable_phoenix2(
a1 string ,
a2 string ,
a3 string ,
a4 string ,
a5 string ,
a6 string ,
a7 string ,
a8 string ,
a9 string ,
a10 string ,
a11 string ,
a12 string ,
a13 string ,
a14 string ,
a15 string ,
a16 string ,
a17 string ,
a18 string ,
a19 string ,
a20 string ,
a21 string ,
a22 string ,
a23 string ,
a24 string ,
a25 string ,
a26 string ,
a27 string ,
a28 string ,
a29 string ,
a30 string ,
a31 string ,
a32 string ,
a33 string ,
a34 string ,
a35 string ,
a36 string ,
a37 string ,
a38 string ,
a39 string ,
a40 string ,
a41 string ,
a42 string ,
a43 string ,
a44 string ,
a45 string ,
a46 string ,
a47 string ,
a48 string ,
a49 string ,
a50 string ,
a51 string ,
a52 string ,
a53 string ,
a54 string ,
a55 string ,
a56 string ,
a57 string ,
a58 string ,
a59 string ,
a60 string ,
a61 string ,
a62 string ,
a63 string ,
a64 string ,
a65 string ,
a66 string ,
a67 string ,
a68 string ,
a69 string ,
a70 string ,
a71 string ,
a72 string ,
a73 string ,
a74 string ,
a75 string ,
a76 string ,
a77 string ,
a78 string ,
a79 string ,
a80 string ,
a81 string ,
a82 string ,
a83 string ,
a84 string ,
a85 string ,
a86 string ,
a87 string ,
a88 string ,
a89 string ,
a90 string ,
a91 string ,
a92 string ,
a93 string ,
a94 string ,
a95 string ,
a96 string ,
a97 string ,
a98 string ,
a99 string ,
a100 string ,
a101 string ,
a102 string ,
a103 string ,
a104 string ,
a105 string ,
a106 string ,
a107 string ,
a108 string ,
a109 string ,
a110 string ,
a111 string ,
a112 string ,
a113 string ,
a114 string ,
a115 string ,
a116 string ,
a117 string ,
a118 string ,
a119 string ,
a120 string ,
a121 string ,
a122 string ,
a123 string ,
a124 string ,
a125 string ,
a126 string ,
a127 string ,
a128 string ,
a129 string ,
a130 string ,
a131 string ,
a132 string ,
a133 string ,
a134 string ,
a135 string ,
a136 string ,
a137 string ,
a138 string ,
a139 string ,
a140 string ,
a141 string ,
a142 string ,
a143 string ,
a144 string ,
a145 string ,
a146 string ,
a147 string ,
a148 string ,
a149 string ,
a150 string ,
a151 string ,
a152 string ,
a153 string ,
a154 string ,
a155 string ,
a156 string ,
a157 string ,
a158 string ,
a159 string ,
a160 string ,
a161 string ,
a162 string ,
a163 string ,
a164 string ,
a165 string ,
a166 string ,
a167 string ,
a168 string ,
a169 timestamp COMMENT '创建时间'
) comment "凤凰外表"
STORED BY
'org.apache.phoenix.hive.PhoenixStorageHandler'
TBLPROPERTIES (
"phoenix.table.name" = "FIN_RISK.T_TONGDUN_PERLOAN_VARIABLE2",
"phoenix.zookeeper.quorum" = "10.201.5.46,10.201.5.47,10.201.5.48",
"phoenix.zookeeper.znode.parent" = "/hbase",
"phoenix.zookeeper.client.port" = "2181",
"phoenix.rowkeys" = "a1" ,
"phoenix.column.mapping" = "a1:a1, a2:a2, a3:a3, a4:a4, a5:a5, a6:a6, a7:a7, a8:a8, a9:a9, a10:a10, a11:a11, a12:a12, a13:a13, a14:a14, a15:a15, a16:a16, a17:a17, a18:a18, a19:a19, a20:a20, a21:a21, a22:a22, a23:a23, a24:a24, a25:a25, a26:a26, a27:a27, a28:a28, a29:a29, a30:a30, a31:a31, a32:a32, a33:a33, a34:a34, a35:a35, a36:a36, a37:a37, a38:a38, a39:a39, a40:a40, a41:a41, a42:a42, a43:a43, a44:a44, a45:a45, a46:a46, a47:a47, a48:a48, a49:a49, a50:a50, a51:a51, a52:a52, a53:a53, a54:a54, a55:a55, a56:a56, a57:a57, a58:a58, a59:a59, a60:a60, a61:a61, a62:a62, a63:a63, a64:a64, a65:a65, a66:a66, a67:a67, a68:a68, a69:a69, a70:a70, a71:a71, a72:a72, a73:a73, a74:a74, a75:a75, a76:a76, a77:a77, a78:a78, a79:a79, a80:a80, a81:a81, a82:a82, a83:a83, a84:a84, a85:a85, a86:a86, a87:a87, a88:a88, a89:a89, a90:a90, a91:a91, a92:a92, a93:a93, a94:a94, a95:a95, a96:a96, a97:a97, a98:a98, a99:a99, a100:a100, a101:a101, a102:a102, a103:a103, a104:a104, a105:a105, a106:a106, a107:a107, a108:a108, a109:a109, a110:a110, a111:a111, a112:a112, a113:a113, a114:a114, a115:a115, a116:a116, a117:a117, a118:a118, a119:a119, a120:a120, a121:a121, a122:a122, a123:a123, a124:a124, a125:a125, a126:a126, a127:a127, a128:a128, a129:a129, a130:a130, a131:a131, a132:a132, a133:a133, a134:a134, a135:a135, a136:a136, a137:a137, a138:a138, a139:a139, a140:a140, a141:a141, a142:a142, a143:a143, a144:a144, a145:a145, a146:a146, a147:a147, a148:a148, a149:a149, a150:a150, a151:a151, a152:a152, a153:a153, a154:a154, a155:a155, a156:a156, a157:a157, a158:a158, a159:a159, a160:a160, a161:a161, a162:a162, a163:a163, a164:a164, a165:a165, a166:a166, a167:a167, a168:a168, a169:a169"
);
select * from fin_risk.bdl_fin_risk_t_tongdun_perloan_variable_phoenix2;
实验可以正常执行;
说明不是因为字段数目导致的报错,而是字段的长度导致的错误。
猜测:
1、代码中进行元数据传输的时候,由于数据类型长度保存问题,导致数据丢失。
CharSequence cs;
cs="biz_no:BIZ_NO,user_id:USER_ID,login_phone:LOGIN_PHONE,id_card:ID_CARD,user_name:USER_NAME,caller_code:CALLER_CODE,order_id:ORDER_ID,code:CODE,idcard_adress_is_belong_high_risk_area:IDCARD_ADRESS_IS_BELONG_HIGH_RISK_AREA,idcard_is_hit_court_broken_list:IDCARD_IS_HIT_COURT_BROKEN_LIST,idcard_is_hit_crime_list:IDCARD_IS_HIT_CRIME_LIST,idcard_is_hit_court_execution_list:IDCARD_IS_HIT_COURT_EXECUTION_LIST,idcard_is_exist_student_loan_overdue_history:IDCARD_IS_EXIST_STUDENT_LOAN_OVERDUE_HISTORY,idcard_is_hit_loan_overdue_list:IDCARD_IS_HIT_LOAN_OVERDUE_LIST,idcard_is_hit_high_risk_attention_list:IDCARD_IS_HIT_HIGH_RISK_ATTENTION_LIST,idcard_is_hit_car_rental_break_list:IDCARD_IS_HIT_CAR_RENTAL_BREAK_LIST,idcard_is_hit_court_closing_list:IDCARD_IS_HIT_COURT_CLOSING_LIST,idcard_name_is_hit_loan_overdue_greylist:IDCARD_NAME_IS_HIT_LOAN_OVERDUE_GREYLIST,idcard_name_is_hit_court_broken_greylist:IDCARD_NAME_IS_HIT_COURT_BROKEN_GREYLIST,idcard_name_is_hit_court_execution_greylist:IDCARD_NAME_IS_HIT_COURT_EXECUTION_GREYLIST,idcard_name_is_hit_court_closing_greylist:IDCARD_NAME_IS_HIT_COURT_CLOSING_GREYLIST,idcard_is_hit_corporation_of_overduecompany_list:IDCARD_IS_HIT_CORPORATION_OF_OVERDUECOMPANY_LIST,idcard_is_hit_purpose_break_rule_riding_list:IDCARD_IS_HIT_PURPOSE_BREAK_RULE_RIDING_LIST,idcard_is_hit_owing_tax_list:IDCARD_IS_HIT_OWING_TAX_LIST,idcard_is_hit_corporation_of_owing_tax_company_list:IDCARD_IS_HIT_CORPORATION_OF_OWING_TAX_COMPANY_LIST,idcard_is_hit_paymoney_after_overdue_list:IDCARD_IS_HIT_PAYMONEY_AFTER_OVERDUE_LIST,phone_is_hit_false_number:PHONE_IS_HIT_FALSE_NUMBER,phone_is_hit_altermate_number:PHONE_IS_HIT_ALTERMATE_NUMBER,phone_is_hit_high_risk_attention_list:PHONE_IS_HIT_HIGH_RISK_ATTENTION_LIST,phone_is_hit_loan_overdue_list:PHONE_IS_HIT_LOAN_OVERDUE_LIST,phone_is_hit_car_rental_break_list:PHONE_IS_HIT_CAR_RENTAL_BREAK_LIST,phone_is_hit_fillin_random:PHONE_IS_HIT_FILLIN_RANDOM,phone_is_hit_corporation_of_overduecompany_list:PHONE_IS_HIT_CORPORATION_OF_OVERDUECOMPANY_LIST,phone_is_hit_aymoney_after_overdue_list:PHONE_IS_HIT_AYMONEY_AFTER_OVERDUE_LIST,idcard_is_related_various_apply_3m:IDCARD_IS_RELATED_VARIOUS_APPLY_3M,apply_is_related_various_idcard_3m:APPLY_IS_RELATED_VARIOUS_IDCARD_3M,bankcard_name_id_is_raleted_various_idcard_3m:BANKCARD_NAME_ID_IS_RALETED_VARIOUS_IDCARD_3M,user_idcard_as_contact_cnt_3m:USER_IDCARD_AS_CONTACT_CNT_3M,user_phone_as_contact_cnt_3m:USER_PHONE_AS_CONTACT_CNT_3M,user_idcard_related_mate_idcard_cnt_3m:USER_IDCARD_RELATED_MATE_IDCARD_CNT_3M,user_idcard_related_mate_phone_cnt_3m:USER_IDCARD_RELATED_MATE_PHONE_CNT_3M,user_idcard_related_father_idcard_cnt_3m:USER_IDCARD_RELATED_FATHER_IDCARD_CNT_3M,user_idcard_related_father_phone_cnt_3m:USER_IDCARD_RELATED_FATHER_PHONE_CNT_3M,user_idcard_related_mother_idcard_cnt_3m:USER_IDCARD_RELATED_MOTHER_IDCARD_CNT_3M,user_idcard_related_mother_phone_cnt_3m:USER_IDCARD_RELATED_MOTHER_PHONE_CNT_3M,user_applyloan_all_organization_cnt_7d:USER_APPLYLOAN_ALL_ORGANIZATION_CNT_7D,idcard_applyloan_all_organization_cnt_7d:IDCARD_APPLYLOAN_ALL_ORGANIZATION_CNT_7D,idcard_applyloan_p2p_organization_cnt_7d:IDCARD_APPLYLOAN_P2P_ORGANIZATION_CNT_7D,idcard_applyloan_onlinestore_organization_cnt_7d:IDCARD_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_7D,idcard_applyloan_carfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_large_consumfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_financiallease_organization_cnt_7d:IDCARD_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_7D,idcard_applyloan_bank_organization_cnt_7d:IDCARD_APPLYLOAN_BANK_ORGANIZATION_CNT_7D,idcard_applyloan_normal_consumfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_bigdatafinancial_organization_cnt_7d:IDCARD_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_guarantor_organization_cnt_7d:IDCARD_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_7D,idcard_applyloan_internatfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_insurance_organization_cnt_7d:IDCARD_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_7D,idcard_applyloan_payment_organization_cnt_7d:IDCARD_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_7D,idcard_applyloan_financing_organization_cnt_7d:IDCARD_APPLYLOAN_FINANCING_ORGANIZATION_CNT_7D,idcard_applyloan_housefinancial_organization_cnt_7d:IDCARD_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_rent_organization_cnt_7d:IDCARD_APPLYLOAN_RENT_ORGANIZATION_CNT_7D,idcard_applyloan_pdl_organization_cnt_7d:IDCARD_APPLYLOAN_PDL_ORGANIZATION_CNT_7D,phone_applyloan_all_organization_cnt_7d:PHONE_APPLYLOAN_ALL_ORGANIZATION_CNT_7D,phone_applyloan_p2p_organization_cnt_7d:PHONE_APPLYLOAN_P2P_ORGANIZATION_CNT_7D,phone_applyloan_onlinestore_organization_cnt_7d:PHONE_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_7D,phone_applyloan_carfinancial_organization_cnt_7d:PHONE_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_large_consumfinancial_organization_cnt_7d:PHONE_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_financiallease_organization_cnt_7d:PHONE_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_7D,phone_applyloan_bank_organization_cnt_7d:PHONE_APPLYLOAN_BANK_ORGANIZATION_CNT_7D,phone_applyloan_normal_consumfinancial_organization_cnt_7d:PHONE_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_bigdatafinancial_organization_cnt_7d:PHONE_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_guarantor_organization_cnt_7d:PHONE_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_7D,phone_applyloan_internatfinancial_organization_cnt_7d:PHONE_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_insurance_organization_cnt_7d:PHONE_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_7D,phone_applyloan_payment_organization_cnt_7d:PHONE_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_7D,phone_applyloan_financing_organization_cnt_7d:PHONE_APPLYLOAN_FINANCING_ORGANIZATION_CNT_7D,phone_applyloan_housefinancial_organization_cnt_7d:PHONE_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_rent_organization_cnt_7d:PHONE_APPLYLOAN_RENT_ORGANIZATION_CNT_7D,phone_applyloan_pdl_organization_cnt_7d:PHONE_APPLYLOAN_PDL_ORGANIZATION_CNT_7D,user_applyloan_all_organization_cnt_1m:USER_APPLYLOAN_ALL_ORGANIZATION_CNT_1M,idcard_applyloan_all_organization_cnt_1m:IDCARD_APPLYLOAN_ALL_ORGANIZATION_CNT_1M,idcard_applyloan_p2p_organization_cnt_1m:IDCARD_APPLYLOAN_P2P_ORGANIZATION_CNT_1M,idcard_applyloan_onlinestore_organization_cnt_1m:IDCARD_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_1M,idcard_applyloan_carfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_large_consumfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_financiallease_organization_cnt_1m:IDCARD_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_1M,idcard_applyloan_bank_organization_cnt_1m:IDCARD_APPLYLOAN_BANK_ORGANIZATION_CNT_1M,idcard_applyloan_normal_consumfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_bigdatafinancial_organization_cnt_1m:IDCARD_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_guarantor_organization_cnt_1m:IDCARD_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_1M,idcard_applyloan_internatfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_insurance_organization_cnt_1m:IDCARD_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_1M,idcard_applyloan_payment_organization_cnt_1m:IDCARD_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_1M,idcard_applyloan_financing_organization_cnt_1m:IDCARD_APPLYLOAN_FINANCING_ORGANIZATION_CNT_1M,idcard_applyloan_housefinancial_organization_cnt_1m:IDCARD_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_rent_organization_cnt_1m:IDCARD_APPLYLOAN_RENT_ORGANIZATION_CNT_1M,idcard_applyloan_pdl_organization_cnt_1m:IDCARD_APPLYLOAN_PDL_ORGANIZATION_CNT_1M,phone_applyloan_all_organization_cnt_1m:PHONE_APPLYLOAN_ALL_ORGANIZATION_CNT_1M,phone_applyloan_p2p_organization_cnt_1m:PHONE_APPLYLOAN_P2P_ORGANIZATION_CNT_1M,phone_applyloan_onlinestore_organization_cnt_1m:PHONE_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_1M,phone_applyloan_carfinancial_organization_cnt_1m:PHONE_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_large_consumfinancial_organization_cnt_1m:PHONE_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_financiallease_organization_cnt_1m:PHONE_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_1M,phone_applyloan_bank_organization_cnt_1m:PHONE_APPLYLOAN_BANK_ORGANIZATION_CNT_1M,phone_applyloan_normal_consumfinancial_organization_cnt_1m:PHONE_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_bigdatafinancial_organization_cnt_1m:PHONE_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_guarantor_organization_cnt_1m:PHONE_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_1M,phone_applyloan_internatfinancial_organization_cnt_1m:PHONE_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_insurance_organization_cnt_1m:PHONE_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_1M,phone_applyloan_payment_organization_cnt_1m:PHONE_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_1M,phone_applyloan_financing_organization_cnt_1m:PHONE_APPLYLOAN_FINANCING_ORGANIZATION_CNT_1M,phone_applyloan_housefinancial_organization_cnt_1m:PHONE_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_rent_organization_cnt_1m:PHONE_APPLYLOAN_RENT_ORGANIZATION_CNT_1M,phone_applyloan_pdl_organization_cnt_1m:PHONE_APPLYLOAN_PDL_ORGANIZATION_CNT_1M,user_applyloan_all_organization_cnt_3m:USER_APPLYLOAN_ALL_ORGANIZATION_CNT_3M,idcard_applyloan_all_organization_cnt_3m:IDCARD_APPLYLOAN_ALL_ORGANIZATION_CNT_3M,idcard_applyloan_p2p_organization_cnt_3m:IDCARD_APPLYLOAN_P2P_ORGANIZATION_CNT_3M,idcard_applyloan_onlinestore_organization_cnt_3m:IDCARD_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_3M,idcard_applyloan_carfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_large_consumfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_financiallease_organization_cnt_3m:IDCARD_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_3M,idcard_applyloan_bank_organization_cnt_3m:IDCARD_APPLYLOAN_BANK_ORGANIZATION_CNT_3M,idcard_applyloan_normal_consumfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_bigdatafinancial_organization_cnt_3m:IDCARD_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_guarantor_organization_cnt_3m:IDCARD_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_3M,idcard_applyloan_internatfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_insurance_organization_cnt_3m:IDCARD_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_3M,idcard_applyloan_payment_organization_cnt_3m:IDCARD_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_3M,idcard_applyloan_financing_organization_cnt_3m:IDCARD_APPLYLOAN_FINANCING_ORGANIZATION_CNT_3M,idcard_applyloan_housefinancial_organization_cnt_3m:IDCARD_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_rent_organization_cnt_3m:IDCARD_APPLYLOAN_RENT_ORGANIZATION_CNT_3M,idcard_applyloan_pdl_organization_cnt_3m:IDCARD_APPLYLOAN_PDL_ORGANIZATION_CNT_3M,phone_applyloan_all_organization_cnt_3m:PHONE_APPLYLOAN_ALL_ORGANIZATION_CNT_3M,phone_applyloan_p2p_organization_cnt_3m:PHONE_APPLYLOAN_P2P_ORGANIZATION_CNT_3M,phone_applyloan_onlinestore_organization_cnt_3m:PHONE_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_3M,phone_applyloan_carfinancial_organization_cnt_3m:PHONE_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_large_consumfinancial_organization_cnt_3m:PHONE_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_financiallease_organization_cnt_3m:PHONE_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_3M,phone_applyloan_bank_organization_cnt_3m:PHONE_APPLYLOAN_BANK_ORGANIZATION_CNT_3M,phone_applyloan_normal_consumfinancial_organization_cnt_3m:PHONE_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_bigdatafinancial_organization_cnt_3m:PHONE_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_guarantor_organization_cnt_3m:PHONE_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_3M,phone_applyloan_internatfinancial_organization_cnt_3m:PHONE_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_insurance_organization_cnt_3m:PHONE_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_3M,phone_applyloan_payment_organization_cnt_3m:PHONE_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_3M,phone_applyloan_financing_organization_cnt_3m:PHONE_APPLYLOAN_FINANCING_ORGANIZATION_CNT_3M,phone_applyloan_housefinancial_organization_cnt_3m:PHONE_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_rent_organization_cnt_3m:PHONE_APPLYLOAN_RENT_ORGANIZATION_CNT_3M,phone_applyloan_pdl_organization_cnt_3m:PHONE_APPLYLOAN_PDL_ORGANIZATION_CNT_3M,user_loan_other_organization_cnt_3m:USER_LOAN_OTHER_ORGANIZATION_CNT_3M,user_loan_own_organization_cnt_3m:USER_LOAN_OWN_ORGANIZATION_CNT_3M,user_is_hit_medium_risk_attention_list:USER_IS_HIT_MEDIUM_RISK_ATTENTION_LIST,user_is_hit_low_risk_attention_list:USER_IS_HIT_LOW_RISK_ATTENTION_LIST,user_applyloan_all_cnt_1hour:USER_APPLYLOAN_ALL_CNT_1HOUR,idcard_applyloan_all_cnt_1hour:IDCARD_APPLYLOAN_ALL_CNT_1HOUR,phone_applyloan_all_cnt_1hour:PHONE_APPLYLOAN_ALL_CNT_1HOUR,user_applyloan_all_cnt_1d:USER_APPLYLOAN_ALL_CNT_1D,idcard_applyloan_all_cnt_1d:IDCARD_APPLYLOAN_ALL_CNT_1D,phone_applyloan_all_cnt_1d:PHONE_APPLYLOAN_ALL_CNT_1D,user_applyloan_all_cnt_7d:USER_APPLYLOAN_ALL_CNT_7D,idcard_applyloan_all_cnt_7d:IDCARD_APPLYLOAN_ALL_CNT_7D,phone_applyloan_all_cnt_7d:PHONE_APPLYLOAN_ALL_CNT_7D,user_applyloan_all_cnt_1m:USER_APPLYLOAN_ALL_CNT_1M,idcard_applyloan_all_cnt_1m:IDCARD_APPLYLOAN_ALL_CNT_1M,phone_applyloan_all_cnt_1m:PHONE_APPLYLOAN_ALL_CNT_1M,idcard_is_hit_attention_list_type:IDCARD_IS_HIT_ATTENTION_LIST_TYPE,phone_is_hit_attention_list_type:PHONE_IS_HIT_ATTENTION_LIST_TYPE,create_time:CREATE_TIME";
System.out.println(cs);
实验可以正常输出;
2、是不是序列化反序列导致数据丢失?
在hive中打开调试模式,输出更加详细的日志:
hive -hiveconf hive.root.logger=DEBUG,console
很明显mapping的值缺少了一部分;
所以可以排除上面的原因,定位到hvie 元数据存储数据丢失;
使用describe 查看表,很明显可以确定对应的问题。
https://www.jianshu.com/p/ccf9c8a43e61 参考这篇文章,修改table_params 的param_value 长度。
param_value 默认4000;修改为20480
实验成功。
ps: 总结遇到错误,一定要打debug日志看看。日志是解决问题的最快方式。