Phoenix 进行hive外表映射bug问题研究

Error while compiling statement: FAILED: RuntimeException org.apache.hadoop.hive.ql.metadata.HiveException: Failed with exception Chunk [idcard_app] is not a valid entryjava.lang.IllegalArgumentException: Chunk [idcard_app] is not a valid entry at com.google.common.base.Preconditions.checkArgument(Preconditions.java:115) at com.google.common.base.Splitter$MapSplitter.split(Splitter.java:452) at org.apache.phoenix.hive.util.ColumnMappingUtils.getColumnMappingMap(ColumnMappingUtils.java:51) at org.apache.phoenix.hive.util.ColumnMappingUtils.getReverseColumnMapping(ColumnMappingUtils.java:62) at org.apache.phoenix.hive.mapreduce.PhoenixResultWritable.setConf(PhoenixResultWritable.java:203) at org.apache.phoenix.hive.mapreduce.PhoenixResultWritable.<init>(PhoenixResultWritable.java:69) at org.apache.phoenix.hive.mapreduce.PhoenixResultWritable.<init>(PhoenixResultWritable.java:74) at org.apache.phoenix.hive.PhoenixSerializer.<init>(PhoenixSerializer.java:80) at org.apache.phoenix.hive.PhoenixSerDe.initialize(PhoenixSerDe.java:86) at org.apache.hadoop.hive.serde2.AbstractSerDe.initialize(AbstractSerDe.java:57) at org.apache.hadoop.hive.serde2.SerDeUtils.initializeSerDeWithoutErrorCheck(SerDeUtils.java:542) at org.apache.hadoop.hive.ql.plan.TableDesc.getDeserializer(TableDesc.java:96) at org.apache.hadoop.hive.ql.exec.FetchOperator.setupOutputObjectInspector(FetchOperator.java:564) at org.apache.hadoop.hive.ql.exec.FetchOperator.initialize(FetchOperator.java:172) at org.apache.hadoop.hive.ql.exec.FetchOperator.<init>(FetchOperator.java:140) at org.apache.hadoop.hive.ql.exec.FetchTask.initialize(FetchTask.java:80) at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:592) at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1356) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1343) at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:185) at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:271) at org.apache.hive.service.cli.operation.Operation.run(Operation.java:337) at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:439) at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:416) at sun.reflect.GeneratedMethodAccessor44.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) at com.sun.proxy.$Proxy22.executeStatementAsync(Unknown Source) at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:282) at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:501) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1313) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1298) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)







实验:建立一个165个字段的Phoenix,hive进行映射Phoenix表出现上面的报错






CREATE TABLE "FIN_RISK"."T_TONGDUN_PERLOAN_VARIABLE2" (
a1 varchar,
a2  varchar,
a3  varchar,
a4  varchar,
a5  varchar,
a6  varchar,
a7  varchar,
a8  varchar,
a9  varchar,
a10 varchar,
a11 varchar,
a12 varchar,
a13 varchar,
a14 varchar,
a15 varchar,
a16 varchar,
a17 varchar,
a18 varchar,
a19 varchar,
a20 varchar,
a21 varchar,
a22 varchar,
a23 varchar,
a24 varchar,
a25 varchar,
a26 varchar,
a27 varchar,
a28 varchar,
a29 varchar,
a30 varchar,
a31 varchar,
a32 varchar,
a33 varchar,
a34 varchar,
a35 varchar,
a36 varchar,
a37 varchar,
a38 varchar,
a39 varchar,
a40 varchar,
a41 varchar,
a42 varchar,
a43 varchar,
a44 varchar,
a45 varchar,
a46 varchar,
a47 varchar,
a48 varchar,
a49 varchar,
a50 varchar,
a51 varchar,
a52 varchar,
a53 varchar,
a54 varchar,
a55 varchar,
a56 varchar,
a57 varchar,
a58 varchar,
a59 varchar,
a60 varchar,
a61 varchar,
a62 varchar,
a63 varchar,
a64 varchar,
a65 varchar,
a66 varchar,
a67 varchar,
a68 varchar,
a69 varchar,
a70 varchar,
a71 varchar,
a72 varchar,
a73 varchar,
a74 varchar,
a75 varchar,
a76 varchar,
a77 varchar,
a78 varchar,
a79 varchar,
a80 varchar,
a81 varchar,
a82 varchar,
a83 varchar,
a84 varchar,
a85 varchar,
a86 varchar,
a87 varchar,
a88 varchar,
a89 varchar,
a90 varchar,
a91 varchar,
a92 varchar,
a93 varchar,
a94 varchar,
a95 varchar,
a96 varchar,
a97 varchar,
a98 varchar,
a99 varchar,
a100    varchar,
a101    varchar,
a102    varchar,
a103    varchar,
a104    varchar,
a105    varchar,
a106    varchar,
a107    varchar,
a108    varchar,
a109    varchar,
a110    varchar,
a111    varchar,
a112    varchar,
a113    varchar,
a114    varchar,
a115    varchar,
a116    varchar,
a117    varchar,
a118    varchar,
a119    varchar,
a120    varchar,
a121    varchar,
a122    varchar,
a123    varchar,
a124    varchar,
a125    varchar,
a126    varchar,
a127    varchar,
a128    varchar,
a129    varchar,
a130    varchar,
a131    varchar,
a132    varchar,
a133    varchar,
a134    varchar,
a135    varchar,
a136    varchar,
a137    varchar,
a138    varchar,
a139    varchar,
a140    varchar,
a141    varchar,
a142    varchar,
a143    varchar,
a144    varchar,
a145    varchar,
a146    varchar,
a147    varchar,
a148    varchar,
a149    varchar,
a150    varchar,
a151    varchar,
a152    varchar,
a153    varchar,
a154    varchar,
a155    varchar,
a156    varchar,
a157    varchar,
a158    varchar,
a159    varchar,
a160    varchar,
a161    varchar,
a162    varchar,
a163    varchar,
a164    varchar,
a165    varchar,
a166    varchar,
a167    varchar,
a168    varchar,
a169    timestamp

  CONSTRAINT PK PRIMARY KEY(a1)
)SALT_BUCKETS=20;







create external table  fin_risk.bdl_fin_risk_t_tongdun_perloan_variable_phoenix2(
a1  string  ,
a2  string  ,
a3  string  ,
a4  string  ,
a5  string  ,
a6  string  ,
a7  string  ,
a8  string  ,
a9  string  ,
a10 string  ,
a11 string  ,
a12 string  ,
a13 string  ,
a14 string  ,
a15 string  ,
a16 string  ,
a17 string  ,
a18 string  ,
a19 string  ,
a20 string  ,
a21 string  ,
a22 string  ,
a23 string  ,
a24 string  ,
a25 string  ,
a26 string  ,
a27 string  ,
a28 string  ,
a29 string  ,
a30 string  ,
a31 string  ,
a32 string  ,
a33 string  ,
a34 string  ,
a35 string  ,
a36 string  ,
a37 string  ,
a38 string  ,
a39 string  ,
a40 string  ,
a41 string  ,
a42 string  ,
a43 string  ,
a44 string  ,
a45 string  ,
a46 string  ,
a47 string  ,
a48 string  ,
a49 string  ,
a50 string  ,
a51 string  ,
a52 string  ,
a53 string  ,
a54 string  ,
a55 string  ,
a56 string  ,
a57 string  ,
a58 string  ,
a59 string  ,
a60 string  ,
a61 string  ,
a62 string  ,
a63 string  ,
a64 string  ,
a65 string  ,
a66 string  ,
a67 string  ,
a68 string  ,
a69 string  ,
a70 string  ,
a71 string  ,
a72 string  ,
a73 string  ,
a74 string  ,
a75 string  ,
a76 string  ,
a77 string  ,
a78 string  ,
a79 string  ,
a80 string  ,
a81 string  ,
a82 string  ,
a83 string  ,
a84 string  ,
a85 string  ,
a86 string  ,
a87 string  ,
a88 string  ,
a89 string  ,
a90 string  ,
a91 string  ,
a92 string  ,
a93 string  ,
a94 string  ,
a95 string  ,
a96 string  ,
a97 string  ,
a98 string  ,
a99 string  ,
a100    string  ,
a101    string  ,
a102    string  ,
a103    string  ,
a104    string  ,
a105    string  ,
a106    string  ,
a107    string  ,
a108    string  ,
a109    string  ,
a110    string  ,
a111    string  ,
a112    string  ,
a113    string  ,
a114    string  ,
a115    string  ,
a116    string  ,
a117    string  ,
a118    string  ,
a119    string  ,
a120    string  ,
a121    string  ,
a122    string  ,
a123    string  ,
a124    string  ,
a125    string  ,
a126    string  ,
a127    string  ,
a128    string  ,
a129    string  ,
a130    string  ,
a131    string  ,
a132    string  ,
a133    string  ,
a134    string  ,
a135    string  ,
a136    string  ,
a137    string  ,
a138    string  ,
a139    string  ,
a140    string  ,
a141    string  ,
a142    string  ,
a143    string  ,
a144    string  ,
a145    string  ,
a146    string  ,
a147    string  ,
a148    string  ,
a149    string  ,
a150    string  ,
a151    string  ,
a152    string  ,
a153    string  ,
a154    string  ,
a155    string  ,
a156    string  ,
a157    string  ,
a158    string  ,
a159    string  ,
a160    string  ,
a161    string  ,
a162    string  ,
a163    string  ,
a164    string  ,
a165    string  ,
a166    string  ,
a167    string  ,
a168    string  ,
a169    timestamp   COMMENT '创建时间'
 ) comment "凤凰外表"
STORED BY
  'org.apache.phoenix.hive.PhoenixStorageHandler' 
TBLPROPERTIES (
"phoenix.table.name" = "FIN_RISK.T_TONGDUN_PERLOAN_VARIABLE2",
"phoenix.zookeeper.quorum" = "10.201.5.46,10.201.5.47,10.201.5.48",
"phoenix.zookeeper.znode.parent" = "/hbase",
"phoenix.zookeeper.client.port" = "2181",
"phoenix.rowkeys" = "a1" ,
"phoenix.column.mapping" = "a1:a1, a2:a2, a3:a3, a4:a4, a5:a5, a6:a6, a7:a7, a8:a8, a9:a9, a10:a10, a11:a11, a12:a12, a13:a13, a14:a14, a15:a15, a16:a16, a17:a17, a18:a18, a19:a19, a20:a20, a21:a21, a22:a22, a23:a23, a24:a24, a25:a25, a26:a26, a27:a27, a28:a28, a29:a29, a30:a30, a31:a31, a32:a32, a33:a33, a34:a34, a35:a35, a36:a36, a37:a37, a38:a38, a39:a39, a40:a40, a41:a41, a42:a42, a43:a43, a44:a44, a45:a45, a46:a46, a47:a47, a48:a48, a49:a49, a50:a50, a51:a51, a52:a52, a53:a53, a54:a54, a55:a55, a56:a56, a57:a57, a58:a58, a59:a59, a60:a60, a61:a61, a62:a62, a63:a63, a64:a64, a65:a65, a66:a66, a67:a67, a68:a68, a69:a69, a70:a70, a71:a71, a72:a72, a73:a73, a74:a74, a75:a75, a76:a76, a77:a77, a78:a78, a79:a79, a80:a80, a81:a81, a82:a82, a83:a83, a84:a84, a85:a85, a86:a86, a87:a87, a88:a88, a89:a89, a90:a90, a91:a91, a92:a92, a93:a93, a94:a94, a95:a95, a96:a96, a97:a97, a98:a98, a99:a99, a100:a100, a101:a101, a102:a102, a103:a103, a104:a104, a105:a105, a106:a106, a107:a107, a108:a108, a109:a109, a110:a110, a111:a111, a112:a112, a113:a113, a114:a114, a115:a115, a116:a116, a117:a117, a118:a118, a119:a119, a120:a120, a121:a121, a122:a122, a123:a123, a124:a124, a125:a125, a126:a126, a127:a127, a128:a128, a129:a129, a130:a130, a131:a131, a132:a132, a133:a133, a134:a134, a135:a135, a136:a136, a137:a137, a138:a138, a139:a139, a140:a140, a141:a141, a142:a142, a143:a143, a144:a144, a145:a145, a146:a146, a147:a147, a148:a148, a149:a149, a150:a150, a151:a151, a152:a152, a153:a153, a154:a154, a155:a155, a156:a156, a157:a157, a158:a158, a159:a159, a160:a160, a161:a161, a162:a162, a163:a163, a164:a164, a165:a165, a166:a166, a167:a167, a168:a168, a169:a169"

);


select * from fin_risk.bdl_fin_risk_t_tongdun_perloan_variable_phoenix2;

实验可以正常执行;

说明不是因为字段数目导致的报错,而是字段的长度导致的错误。

猜测:

1、代码中进行元数据传输的时候,由于数据类型长度保存问题,导致数据丢失。

CharSequence cs;
cs="biz_no:BIZ_NO,user_id:USER_ID,login_phone:LOGIN_PHONE,id_card:ID_CARD,user_name:USER_NAME,caller_code:CALLER_CODE,order_id:ORDER_ID,code:CODE,idcard_adress_is_belong_high_risk_area:IDCARD_ADRESS_IS_BELONG_HIGH_RISK_AREA,idcard_is_hit_court_broken_list:IDCARD_IS_HIT_COURT_BROKEN_LIST,idcard_is_hit_crime_list:IDCARD_IS_HIT_CRIME_LIST,idcard_is_hit_court_execution_list:IDCARD_IS_HIT_COURT_EXECUTION_LIST,idcard_is_exist_student_loan_overdue_history:IDCARD_IS_EXIST_STUDENT_LOAN_OVERDUE_HISTORY,idcard_is_hit_loan_overdue_list:IDCARD_IS_HIT_LOAN_OVERDUE_LIST,idcard_is_hit_high_risk_attention_list:IDCARD_IS_HIT_HIGH_RISK_ATTENTION_LIST,idcard_is_hit_car_rental_break_list:IDCARD_IS_HIT_CAR_RENTAL_BREAK_LIST,idcard_is_hit_court_closing_list:IDCARD_IS_HIT_COURT_CLOSING_LIST,idcard_name_is_hit_loan_overdue_greylist:IDCARD_NAME_IS_HIT_LOAN_OVERDUE_GREYLIST,idcard_name_is_hit_court_broken_greylist:IDCARD_NAME_IS_HIT_COURT_BROKEN_GREYLIST,idcard_name_is_hit_court_execution_greylist:IDCARD_NAME_IS_HIT_COURT_EXECUTION_GREYLIST,idcard_name_is_hit_court_closing_greylist:IDCARD_NAME_IS_HIT_COURT_CLOSING_GREYLIST,idcard_is_hit_corporation_of_overduecompany_list:IDCARD_IS_HIT_CORPORATION_OF_OVERDUECOMPANY_LIST,idcard_is_hit_purpose_break_rule_riding_list:IDCARD_IS_HIT_PURPOSE_BREAK_RULE_RIDING_LIST,idcard_is_hit_owing_tax_list:IDCARD_IS_HIT_OWING_TAX_LIST,idcard_is_hit_corporation_of_owing_tax_company_list:IDCARD_IS_HIT_CORPORATION_OF_OWING_TAX_COMPANY_LIST,idcard_is_hit_paymoney_after_overdue_list:IDCARD_IS_HIT_PAYMONEY_AFTER_OVERDUE_LIST,phone_is_hit_false_number:PHONE_IS_HIT_FALSE_NUMBER,phone_is_hit_altermate_number:PHONE_IS_HIT_ALTERMATE_NUMBER,phone_is_hit_high_risk_attention_list:PHONE_IS_HIT_HIGH_RISK_ATTENTION_LIST,phone_is_hit_loan_overdue_list:PHONE_IS_HIT_LOAN_OVERDUE_LIST,phone_is_hit_car_rental_break_list:PHONE_IS_HIT_CAR_RENTAL_BREAK_LIST,phone_is_hit_fillin_random:PHONE_IS_HIT_FILLIN_RANDOM,phone_is_hit_corporation_of_overduecompany_list:PHONE_IS_HIT_CORPORATION_OF_OVERDUECOMPANY_LIST,phone_is_hit_aymoney_after_overdue_list:PHONE_IS_HIT_AYMONEY_AFTER_OVERDUE_LIST,idcard_is_related_various_apply_3m:IDCARD_IS_RELATED_VARIOUS_APPLY_3M,apply_is_related_various_idcard_3m:APPLY_IS_RELATED_VARIOUS_IDCARD_3M,bankcard_name_id_is_raleted_various_idcard_3m:BANKCARD_NAME_ID_IS_RALETED_VARIOUS_IDCARD_3M,user_idcard_as_contact_cnt_3m:USER_IDCARD_AS_CONTACT_CNT_3M,user_phone_as_contact_cnt_3m:USER_PHONE_AS_CONTACT_CNT_3M,user_idcard_related_mate_idcard_cnt_3m:USER_IDCARD_RELATED_MATE_IDCARD_CNT_3M,user_idcard_related_mate_phone_cnt_3m:USER_IDCARD_RELATED_MATE_PHONE_CNT_3M,user_idcard_related_father_idcard_cnt_3m:USER_IDCARD_RELATED_FATHER_IDCARD_CNT_3M,user_idcard_related_father_phone_cnt_3m:USER_IDCARD_RELATED_FATHER_PHONE_CNT_3M,user_idcard_related_mother_idcard_cnt_3m:USER_IDCARD_RELATED_MOTHER_IDCARD_CNT_3M,user_idcard_related_mother_phone_cnt_3m:USER_IDCARD_RELATED_MOTHER_PHONE_CNT_3M,user_applyloan_all_organization_cnt_7d:USER_APPLYLOAN_ALL_ORGANIZATION_CNT_7D,idcard_applyloan_all_organization_cnt_7d:IDCARD_APPLYLOAN_ALL_ORGANIZATION_CNT_7D,idcard_applyloan_p2p_organization_cnt_7d:IDCARD_APPLYLOAN_P2P_ORGANIZATION_CNT_7D,idcard_applyloan_onlinestore_organization_cnt_7d:IDCARD_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_7D,idcard_applyloan_carfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_large_consumfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_financiallease_organization_cnt_7d:IDCARD_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_7D,idcard_applyloan_bank_organization_cnt_7d:IDCARD_APPLYLOAN_BANK_ORGANIZATION_CNT_7D,idcard_applyloan_normal_consumfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_bigdatafinancial_organization_cnt_7d:IDCARD_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_guarantor_organization_cnt_7d:IDCARD_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_7D,idcard_applyloan_internatfinancial_organization_cnt_7d:IDCARD_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_insurance_organization_cnt_7d:IDCARD_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_7D,idcard_applyloan_payment_organization_cnt_7d:IDCARD_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_7D,idcard_applyloan_financing_organization_cnt_7d:IDCARD_APPLYLOAN_FINANCING_ORGANIZATION_CNT_7D,idcard_applyloan_housefinancial_organization_cnt_7d:IDCARD_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_7D,idcard_applyloan_rent_organization_cnt_7d:IDCARD_APPLYLOAN_RENT_ORGANIZATION_CNT_7D,idcard_applyloan_pdl_organization_cnt_7d:IDCARD_APPLYLOAN_PDL_ORGANIZATION_CNT_7D,phone_applyloan_all_organization_cnt_7d:PHONE_APPLYLOAN_ALL_ORGANIZATION_CNT_7D,phone_applyloan_p2p_organization_cnt_7d:PHONE_APPLYLOAN_P2P_ORGANIZATION_CNT_7D,phone_applyloan_onlinestore_organization_cnt_7d:PHONE_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_7D,phone_applyloan_carfinancial_organization_cnt_7d:PHONE_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_large_consumfinancial_organization_cnt_7d:PHONE_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_financiallease_organization_cnt_7d:PHONE_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_7D,phone_applyloan_bank_organization_cnt_7d:PHONE_APPLYLOAN_BANK_ORGANIZATION_CNT_7D,phone_applyloan_normal_consumfinancial_organization_cnt_7d:PHONE_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_bigdatafinancial_organization_cnt_7d:PHONE_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_guarantor_organization_cnt_7d:PHONE_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_7D,phone_applyloan_internatfinancial_organization_cnt_7d:PHONE_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_insurance_organization_cnt_7d:PHONE_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_7D,phone_applyloan_payment_organization_cnt_7d:PHONE_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_7D,phone_applyloan_financing_organization_cnt_7d:PHONE_APPLYLOAN_FINANCING_ORGANIZATION_CNT_7D,phone_applyloan_housefinancial_organization_cnt_7d:PHONE_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_7D,phone_applyloan_rent_organization_cnt_7d:PHONE_APPLYLOAN_RENT_ORGANIZATION_CNT_7D,phone_applyloan_pdl_organization_cnt_7d:PHONE_APPLYLOAN_PDL_ORGANIZATION_CNT_7D,user_applyloan_all_organization_cnt_1m:USER_APPLYLOAN_ALL_ORGANIZATION_CNT_1M,idcard_applyloan_all_organization_cnt_1m:IDCARD_APPLYLOAN_ALL_ORGANIZATION_CNT_1M,idcard_applyloan_p2p_organization_cnt_1m:IDCARD_APPLYLOAN_P2P_ORGANIZATION_CNT_1M,idcard_applyloan_onlinestore_organization_cnt_1m:IDCARD_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_1M,idcard_applyloan_carfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_large_consumfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_financiallease_organization_cnt_1m:IDCARD_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_1M,idcard_applyloan_bank_organization_cnt_1m:IDCARD_APPLYLOAN_BANK_ORGANIZATION_CNT_1M,idcard_applyloan_normal_consumfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_bigdatafinancial_organization_cnt_1m:IDCARD_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_guarantor_organization_cnt_1m:IDCARD_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_1M,idcard_applyloan_internatfinancial_organization_cnt_1m:IDCARD_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_insurance_organization_cnt_1m:IDCARD_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_1M,idcard_applyloan_payment_organization_cnt_1m:IDCARD_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_1M,idcard_applyloan_financing_organization_cnt_1m:IDCARD_APPLYLOAN_FINANCING_ORGANIZATION_CNT_1M,idcard_applyloan_housefinancial_organization_cnt_1m:IDCARD_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_1M,idcard_applyloan_rent_organization_cnt_1m:IDCARD_APPLYLOAN_RENT_ORGANIZATION_CNT_1M,idcard_applyloan_pdl_organization_cnt_1m:IDCARD_APPLYLOAN_PDL_ORGANIZATION_CNT_1M,phone_applyloan_all_organization_cnt_1m:PHONE_APPLYLOAN_ALL_ORGANIZATION_CNT_1M,phone_applyloan_p2p_organization_cnt_1m:PHONE_APPLYLOAN_P2P_ORGANIZATION_CNT_1M,phone_applyloan_onlinestore_organization_cnt_1m:PHONE_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_1M,phone_applyloan_carfinancial_organization_cnt_1m:PHONE_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_large_consumfinancial_organization_cnt_1m:PHONE_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_financiallease_organization_cnt_1m:PHONE_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_1M,phone_applyloan_bank_organization_cnt_1m:PHONE_APPLYLOAN_BANK_ORGANIZATION_CNT_1M,phone_applyloan_normal_consumfinancial_organization_cnt_1m:PHONE_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_bigdatafinancial_organization_cnt_1m:PHONE_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_guarantor_organization_cnt_1m:PHONE_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_1M,phone_applyloan_internatfinancial_organization_cnt_1m:PHONE_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_insurance_organization_cnt_1m:PHONE_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_1M,phone_applyloan_payment_organization_cnt_1m:PHONE_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_1M,phone_applyloan_financing_organization_cnt_1m:PHONE_APPLYLOAN_FINANCING_ORGANIZATION_CNT_1M,phone_applyloan_housefinancial_organization_cnt_1m:PHONE_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_1M,phone_applyloan_rent_organization_cnt_1m:PHONE_APPLYLOAN_RENT_ORGANIZATION_CNT_1M,phone_applyloan_pdl_organization_cnt_1m:PHONE_APPLYLOAN_PDL_ORGANIZATION_CNT_1M,user_applyloan_all_organization_cnt_3m:USER_APPLYLOAN_ALL_ORGANIZATION_CNT_3M,idcard_applyloan_all_organization_cnt_3m:IDCARD_APPLYLOAN_ALL_ORGANIZATION_CNT_3M,idcard_applyloan_p2p_organization_cnt_3m:IDCARD_APPLYLOAN_P2P_ORGANIZATION_CNT_3M,idcard_applyloan_onlinestore_organization_cnt_3m:IDCARD_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_3M,idcard_applyloan_carfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_large_consumfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_financiallease_organization_cnt_3m:IDCARD_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_3M,idcard_applyloan_bank_organization_cnt_3m:IDCARD_APPLYLOAN_BANK_ORGANIZATION_CNT_3M,idcard_applyloan_normal_consumfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_bigdatafinancial_organization_cnt_3m:IDCARD_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_guarantor_organization_cnt_3m:IDCARD_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_3M,idcard_applyloan_internatfinancial_organization_cnt_3m:IDCARD_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_insurance_organization_cnt_3m:IDCARD_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_3M,idcard_applyloan_payment_organization_cnt_3m:IDCARD_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_3M,idcard_applyloan_financing_organization_cnt_3m:IDCARD_APPLYLOAN_FINANCING_ORGANIZATION_CNT_3M,idcard_applyloan_housefinancial_organization_cnt_3m:IDCARD_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_3M,idcard_applyloan_rent_organization_cnt_3m:IDCARD_APPLYLOAN_RENT_ORGANIZATION_CNT_3M,idcard_applyloan_pdl_organization_cnt_3m:IDCARD_APPLYLOAN_PDL_ORGANIZATION_CNT_3M,phone_applyloan_all_organization_cnt_3m:PHONE_APPLYLOAN_ALL_ORGANIZATION_CNT_3M,phone_applyloan_p2p_organization_cnt_3m:PHONE_APPLYLOAN_P2P_ORGANIZATION_CNT_3M,phone_applyloan_onlinestore_organization_cnt_3m:PHONE_APPLYLOAN_ONLINESTORE_ORGANIZATION_CNT_3M,phone_applyloan_carfinancial_organization_cnt_3m:PHONE_APPLYLOAN_CARFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_large_consumfinancial_organization_cnt_3m:PHONE_APPLYLOAN_LARGE_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_financiallease_organization_cnt_3m:PHONE_APPLYLOAN_FINANCIALLEASE_ORGANIZATION_CNT_3M,phone_applyloan_bank_organization_cnt_3m:PHONE_APPLYLOAN_BANK_ORGANIZATION_CNT_3M,phone_applyloan_normal_consumfinancial_organization_cnt_3m:PHONE_APPLYLOAN_NORMAL_CONSUMFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_bigdatafinancial_organization_cnt_3m:PHONE_APPLYLOAN_BIGDATAFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_guarantor_organization_cnt_3m:PHONE_APPLYLOAN_GUARANTOR_ORGANIZATION_CNT_3M,phone_applyloan_internatfinancial_organization_cnt_3m:PHONE_APPLYLOAN_INTERNATFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_insurance_organization_cnt_3m:PHONE_APPLYLOAN_INSURANCE_ORGANIZATION_CNT_3M,phone_applyloan_payment_organization_cnt_3m:PHONE_APPLYLOAN_PAYMENT_ORGANIZATION_CNT_3M,phone_applyloan_financing_organization_cnt_3m:PHONE_APPLYLOAN_FINANCING_ORGANIZATION_CNT_3M,phone_applyloan_housefinancial_organization_cnt_3m:PHONE_APPLYLOAN_HOUSEFINANCIAL_ORGANIZATION_CNT_3M,phone_applyloan_rent_organization_cnt_3m:PHONE_APPLYLOAN_RENT_ORGANIZATION_CNT_3M,phone_applyloan_pdl_organization_cnt_3m:PHONE_APPLYLOAN_PDL_ORGANIZATION_CNT_3M,user_loan_other_organization_cnt_3m:USER_LOAN_OTHER_ORGANIZATION_CNT_3M,user_loan_own_organization_cnt_3m:USER_LOAN_OWN_ORGANIZATION_CNT_3M,user_is_hit_medium_risk_attention_list:USER_IS_HIT_MEDIUM_RISK_ATTENTION_LIST,user_is_hit_low_risk_attention_list:USER_IS_HIT_LOW_RISK_ATTENTION_LIST,user_applyloan_all_cnt_1hour:USER_APPLYLOAN_ALL_CNT_1HOUR,idcard_applyloan_all_cnt_1hour:IDCARD_APPLYLOAN_ALL_CNT_1HOUR,phone_applyloan_all_cnt_1hour:PHONE_APPLYLOAN_ALL_CNT_1HOUR,user_applyloan_all_cnt_1d:USER_APPLYLOAN_ALL_CNT_1D,idcard_applyloan_all_cnt_1d:IDCARD_APPLYLOAN_ALL_CNT_1D,phone_applyloan_all_cnt_1d:PHONE_APPLYLOAN_ALL_CNT_1D,user_applyloan_all_cnt_7d:USER_APPLYLOAN_ALL_CNT_7D,idcard_applyloan_all_cnt_7d:IDCARD_APPLYLOAN_ALL_CNT_7D,phone_applyloan_all_cnt_7d:PHONE_APPLYLOAN_ALL_CNT_7D,user_applyloan_all_cnt_1m:USER_APPLYLOAN_ALL_CNT_1M,idcard_applyloan_all_cnt_1m:IDCARD_APPLYLOAN_ALL_CNT_1M,phone_applyloan_all_cnt_1m:PHONE_APPLYLOAN_ALL_CNT_1M,idcard_is_hit_attention_list_type:IDCARD_IS_HIT_ATTENTION_LIST_TYPE,phone_is_hit_attention_list_type:PHONE_IS_HIT_ATTENTION_LIST_TYPE,create_time:CREATE_TIME";
System.out.println(cs);

实验可以正常输出;

2、是不是序列化反序列导致数据丢失?

Phoenix 进行hive外表映射bug问题研究

在hive中打开调试模式,输出更加详细的日志:

hive -hiveconf hive.root.logger=DEBUG,console

Phoenix 进行hive外表映射bug问题研究

很明显mapping的值缺少了一部分;

所以可以排除上面的原因,定位到hvie 元数据存储数据丢失;

使用describe 查看表,很明显可以确定对应的问题。

Phoenix 进行hive外表映射bug问题研究

 

Phoenix 进行hive外表映射bug问题研究Phoenix 进行hive外表映射bug问题研究https://www.jianshu.com/p/ccf9c8a43e61 参考这篇文章,修改table_params 的param_value 长度。

param_value 默认4000;修改为20480

Phoenix 进行hive外表映射bug问题研究

 

实验成功。

ps: 总结遇到错误,一定要打debug日志看看。日志是解决问题的最快方式。