OceanBase 数据库集群执行 SQL 出现卡住情况，OceanBase 数据库日志内出现 -4138 重试行为的原因和解决方法

[2024-06-03 10:14:38.796565] WARN  [STORAGE] get_read_tables (ob_table_store.cpp:244) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=3] [dc=0] cannot found specified version(ret=-4138, snapshot_version=1717291818096365, PRETTY_TS(*this)={ObTableStore_Pretty:this:0x7f49760b6a40, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, uptime:1717372827910446, table_count:3, start_pos:1, inc_pos:2, is_ready_for_read:true, replay_tables:[{table_type:1, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717284610768077, base_version:0, snapshot_version:1717284610768077}, log_ts_range:{start_log_ts:0, end_log_ts:0, max_log_ts:0}, version:"909-0-0"}, {table_type:1, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717371010815071, base_version:0, snapshot_version:1717371010815071}, log_ts_range:{start_log_ts:0, end_log_ts:0, max_log_ts:0}, version:"910-0-0"}, {table_type:7, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717369234205167, base_version:1717284620127675, snapshot_version:1717371017041293}, log_ts_range:{start_log_ts:1717284620127675, end_log_ts:1717371017041293, max_log_ts:1717371017041293}, version:"0-47939020-0"}], multi_version_start:1717369234205167, pg_memtable_mgr:{[{i:109, table_key:{table_type:0, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717371017041293, base_version:1717371017041293, snapshot_version:9223372036854775807}, log_ts_range:{start_log_ts:1717371017041293, end_log_ts:9223372036854775807, max_log_ts:0}, version:"0-0-0"}, version:"0-0-0", ref:1}, ]}, complement_minor_sstable:{ObITable:{this:0x7ee8d8b26840, key:{table_type:5, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717369234205167, base_version:1717284620127675, snapshot_version:1717371017041293}, log_ts_range:{start_log_ts:1717284620127675, end_log_ts:1717371017041293, max_log_ts:1717371017041293}, version:"0-47939366-0"}, ref_cnt:1, upper_trans_version:1717371017041293, timestamp:1717284620215826}, this:0x7ee8d8b26840, status:4, meta:{ObSSTableBaseMeta:{index_id:1101710651031672, row_count:0, occupy_size:0, data_checksum:0, data_version:205898009160974336, rowkey_column_count:3, table_type:0, index_type:0, macro_block_count:0, use_old_macro_block_count:0, column_cnt:11, column_metas:[{column_id:18, column_default_checksum:0, column_checksum:0}, {column_id:19, column_default_checksum:0, column_checksum:0}, {column_id:20, column_default_checksum:2265377830, column_checksum:0}, {column_id:16, column_default_checksum:0, column_checksum:0}, {column_id:17, column_default_checksum:0, column_checksum:0}, {column_id:21, column_default_checksum:0, column_checksum:0}, {column_id:22, column_default_checksum:0, column_checksum:0}, {column_id:23, column_default_checksum:0, column_checksum:0}, {column_id:24, column_default_checksum:0, column_checksum:0}, {column_id:25, column_default_checksum:2265377830, column_checksum:0}, {column_id:26, column_default_checksum:2265377830, column_checksum:0}], lob_macro_block_count:0, lob_use_old_macro_block_count:0, build_on_snapshot:false, create_index_base_version:0, schema_version:1712634634083080, progressive_merge_start_version:0, progressive_merge_end_version:0, create_snapshot_version:0, checksum_method:0, progressive_merge_round:0, progressive_merge_step:0, table_mode:{table_mode_flag:0, pk_mode:0}, pg_key:{tid:1101710651031672, partition_id:0, part_cnt:0}, logical_data_version:1717371017041293, multi_version_rowkey_type:2, contain_uncommitted_row:false, upper_trans_version:1717371017041293, max_merged_trans_version:1717371017041293, end_log_id:0, sstable_format_version:6, has_compact_row:true}, file_ctx:{file_system_type:3, file_type:3, block_count_per_file:0, file_list:[]}, lob_file_ctx:{file_system_type:3, file_type:3, block_count_per_file:0, file_list:[]}, macro_block_idx_array:[], lob_macro_block_idx_array:[], bloom_filter_block_id:[0-9223372036854775807](ver=1,mode=0,seq=0), bloom_filter_block_id_in_files:0, bloom_filter_file_ctx:{file_system_type:3, file_type:3, block_count_per_file:0, file_list:[]}}, dump_memtable_timestamp:1717284620215826}, table_array:{
[10526] [ xxxxx-xxxxx-xxxxx-xxxxx ]  pos      table_type version  base_version     multi_ver_start  snapshot_version max_merge_ver    upper_trans_ver     start_log_ts     end_log_ts       max_log_ts       ref  flag cpat 
[10526] [ xxxxx-xxxxx-xxxxx-xxxxx ]           MAJOR      909      0                1717284610768077 1717284610768077 1717284610768077 1717284610768077    0                0                0                1    0    1    
[10526] [ xxxxx-xxxxx-xxxxx-xxxxx ]  start    MAJOR      910      0                1717371010815071 1717371010815071 1717371010815071 1717371010815071    0                0                0                1    0    1    
[10526] [ xxxxx-xxxxx-xxxxx-xxxxx ]  inc      MINI       0        1717284620127675 1717369234205167 1717371017041293 1717371017041293 1717371017041293    1717284620127675 1717371017041293 1717371017041293 1    0    1    
[10526] [ xxxxx-xxxxx-xxxxx-xxxxx ]  comple   COMPLEMENT 0        1717284620127675 1717369234205167 1717371017041293 1717371017041293 1717371017041293    1717284620127675 1717371017041293 1717371017041293 1    0    1    }})
[2024-06-03 10:14:38.796578] WARN  [STORAGE] get_read_tables (ob_multi_version_table_store.cpp:496) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=12] [dc=0] failed to get read tables(ret=-4138)
[2024-06-03 10:14:38.796584] WARN  [STORAGE] get_read_tables (ob_partition_store.cpp:1075) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=5] [dc=0] failed to get read tables(ret=-4138, handle={table_count:0, []}, table_store=0x7f478cd63260)
[2024-06-03 10:14:38.796591] WARN  [STORAGE] get_recovery_point_table_data (ob_reserved_data_mgr.cpp:1422) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=3] [dc=0] can not find corresponding recovery point data(ret=-4018, snapshot_version=1717291818096365, this={pg_key:{tid:1101710651031672, partition_id:0, part_cnt:0}, recovery_point_data count:0[]})
[2024-06-03 10:14:38.796594] WARN  [STORAGE] get_restore_point_read_tables (ob_reserved_data_mgr.cpp:2081) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=3] [dc=0] get recovery point table data failed(ret=-4018, table_id=1101710651031672, snapshot_version=1717291818096365)
[2024-06-03 10:14:38.796598] WARN  [STORAGE] prepare_read_tables (ob_multiple_merge.cpp:1442) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=3] [dc=0] fail to get read tables(ret=-4138)
[2024-06-03 10:14:38.796601] WARN  [STORAGE] init (ob_multiple_merge.cpp:189) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=2] [dc=0] fail to prepare read tables(ret=-4138)

[2024-06-03 10:14:40.380711] WARN  [STORAGE] get_read_tables (ob_table_store.cpp:244) [10528][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=2] [dc=0] cannot found specified version(ret=-4138, snapshot_version=1717291818096365, PRETTY_TS(*this)={ObTableStore_Pretty:this:0x7f49760b6a40, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, uptime:1717372827910446, table_count:3, start_pos:1, inc_pos:2, is_ready_for_read:true, replay_tables:[{table_type:1, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717284610768077, base_version:0, snapshot_version:1717284610768077}, log_ts_range:{start_log_ts:0, end_log_ts:0, max_log_ts:0}, version:"909-0-0"}, {table_type:1, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717371010815071, base_version:0, snapshot_version:1717371010815071}, log_ts_range:{start_log_ts:0, end_log_ts:0, max_log_ts:0}, version:"910-0-0"}, {table_type:7, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717369234205167, base_version:1717284620127675, snapshot_version:1717371017041293}, log_ts_range:{start_log_ts:1717284620127675, end_log_ts:1717371017041293, max_log_ts:1717371017041293}, version:"0-47939020-0"}], multi_version_start:1717369234205167, pg_memtable_mgr:{[{i:109, table_key:{table_type:0, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717371017041293, base_version:1717371017041293, snapshot_version:9223372036854775807}, log_ts_range:{start_log_ts:1717371017041293, end_log_ts:9223372036854775807, max_log_ts:0}, version:"0-0-0"}, version:"0-0-0", ref:1}, ]}, complement_minor_sstable:{ObITable:{this:0x7ee8d8b26840, key:{table_type:5, pkey:{tid:1101710651031672, partition_id:0, part_cnt:0}, table_id:1101710651031672, trans_version_range:{multi_version_start:1717369234205167, base_version:1717284620127675, snapshot_version:1717371017041293}, log_ts_range:{start_log_ts:1717284620127675, end_log_ts:1717371017041293, max_log_ts:1717371017041293}, version:"0-47939366-0"}, ref_cnt:1, upper_trans_version:1717371017041293, timestamp:1717284620215826}, this:0x7ee8d8b26840, status:4, meta:{ObSSTableBaseMeta:{index_id:1101710651031672, row_count:0, occupy_size:0, data_checksum:0, data_version:205898009160974336, rowkey_column_count:3, table_type:0, index_type:0, macro_block_count:0, use_old_macro_block_count:0, column_cnt:11, column_metas:[{column_id:18, column_default_checksum:0, column_checksum:0}, {column_id:19, column_default_checksum:0, column_checksum:0}, {column_id:20, column_default_checksum:2265377830, column_checksum:0}, {column_id:16, column_default_checksum:0, column_checksum:0}, {column_id:17, column_default_checksum:0, column_checksum:0}, {column_id:21, column_default_checksum:0, column_checksum:0}, {column_id:22, column_default_checksum:0, column_checksum:0}, {column_id:23, column_default_checksum:0, column_checksum:0}, {column_id:24, column_default_checksum:0, column_checksum:0}, {column_id:25, column_default_checksum:2265377830, column_checksum:0}, {column_id:26, column_default_checksum:2265377830, column_checksum:0}], lob_macro_block_count:0, lob_use_old_macro_block_count:0, build_on_snapshot:false, create_index_base_version:0, schema_version:1712634634083080, progressive_merge_start_version:0, progressive_merge_end_version:0, create_snapshot_version:0, checksum_method:0, progressive_merge_round:0, progressive_merge_step:0, table_mode:{table_mode_flag:0, pk_mode:0}, pg_key:{tid:1101710651031672, partition_id:0, part_cnt:0}, logical_data_version:1717371017041293, multi_version_rowkey_type:2, contain_uncommitted_row:false, upper_trans_version:1717371017041293, max_merged_trans_version:1717371017041293, end_log_id:0, sstable_format_version:6, has_compact_row:true}, file_ctx:{file_system_type:3, file_type:3, block_count_per_file:0, file_list:[]}, lob_file_ctx:{file_system_type:3, file_type:3, block_count_per_file:0, file_list:[]}, macro_block_idx_array:[], lob_macro_block_idx_array:[], bloom_filter_block_id:[0-9223372036854775807](ver=1,mode=0,seq=0), bloom_filter_block_id_in_files:0, bloom_filter_file_ctx:{file_system_type:3, file_type:3, block_count_per_file:0, file_list:[]}}, dump_memtable_timestamp:1717284620215826}, table_array:{
[2024-06-03 10:14:38.796578] WARN  [STORAGE] get_read_tables (ob_multi_version_table_store.cpp:496) [10526][0][xxxxx-xxxxx-xxxxx-xxxxx] [lt=12] [dc=0] failed to get read tables(ret=-4138)