runit

#!/usr/bin/env bash
bash -n "$0" | exit 1

source ${TESTSROOTDIR}/tools/runit_common.sh
source ${TESTSROOTDIR}/tools/cluster_utils.sh

# Test time partition new truncate rollout
################################################################################


# args
# <dbname>
dbname=$1

OUT="run.log"

rm $OUT 2>/dev/null
touch $OUT

cmd="cdb2sql ${CDB2_OPTIONS} $dbname default"
cmdt="cdb2sql -tabs ${CDB2_OPTIONS} $dbname default"
master=`$cmdt 'exec procedure sys.cmd.send("bdb cluster")' | grep MASTER | cut -f1 -d":" | tr -d '[:space:]'`
cmdm="cdb2sql ${CDB2_OPTIONS} --host $master $dbname default"
cmdmt="cdb2sql -tabs ${CDB2_OPTIONS} --host $master $dbname default"


function timepart_stats
{
    # check the current partitions 
    echo $cmdt "exec procedure sys.cmd.send('partitions')" | egrep -v "STARTTIME|LOW|HIGH|SOURCE_ID" >> $OUT
    $cmdt "exec procedure sys.cmd.send('partitions')" | egrep -v "STARTTIME|LOW|HIGH|SOURCE_ID" >> $OUT
    if (( $? != 0 )) ; then
        echo "FAILURE"
        exit 1
    fi

    echo $cmd "select name, period, retention, nshards, version,shard0name from comdb2_timepartitions " >> $OUT
    $cmd "select name, period, retention, nshards, version,shard0name from comdb2_timepartitions " >> $OUT
    if (( $? != 0 )) ; then
        echo "FAILURE"
        exit 1
    fi

    echo $cmd "select name, shardname from comdb2_timepartshards" >> $OUT
    $cmd "select name, shardname from comdb2_timepartshards" >> $OUT
    if (( $? != 0 )) ; then
        echo "FAILURE"
        exit 1
    fi
   
    sleep $1
    
    echo cdb2sql ${CDB2_OPTIONS} --host MASTER $dbname default  "select name, arg1, arg2, arg3 from comdb2_timepartevents order by 1, 2" >> $OUT
    $cmdm "select name, arg1, arg2, arg3 from comdb2_timepartevents order by 1, 2" >> $OUT
    if (( $? != 0 )) ; then
        echo "FAILURE"
        exit 1
    fi
}

function header
{
    echo "TEST $1"
    echo "$2"
    echo "TEST $1" >> $OUT
    echo "$2" >> $OUT
}
timepart_stats 0

VT="t"

# TEST 1
# create new partition in past, make sure start is in the future
header 1 "create new partition in past, make sure start is in the future"
starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()+60-24*3600)'`
echo $cmd "CREATE TABLE ${VT}(a int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "CREATE TABLE ${VT}(a int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'"
if (( $? != 0 )) ; then
   echo "FAILURE"
   exit 1
fi

timepart_stats 0

starttime=`$cmdmt "select e.epoch from comdb2_cron_events e,comdb2_timepartitions t on e.sourceid=t.sourceid where e.type='timepart_cron' and t.name='${VT}'"`
now=`date "+%s"`

if [[ $starttime -lt $now ]] ; then
    echo "FAIL TEST1: Start time for the new partition is in the past $starttime < $now"
    exit 1
fi

# TEST 2.
# create dup partition
header 2 "create dup partition"
starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()+60)'`
echo $cmd "CREATE TABLE ${VT}(a int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "CREATE TABLE ${VT}(a int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'"
if (( $? == 0 )) ; then
    echo "FAILURE dup partition"
    exit 1
fi

timepart_stats 0

# TEST 3.
# insert some rows in current partition, wait for rollout, insert more, check row location
# (it is enough to wait for the already created "t" partition to rollout in one minute)
header 3 "insert some rows in current partition, wait for rollout, insert more, check row location"
echo $cmd "insert into '\$1_A2620AE4' values (1), (2), (3)"
$cmd "insert into '\$1_A2620AE4' values (1), (2), (3)" >> $OUT
if (( $? != 0 )) ; then
    echo "FAILURE insert oldest shard"
    exit 1
fi
echo $cmd "insert into ${VT} values (10), (20), (30)"
$cmd "insert into ${VT} values (10), (20), (30)"
if (( $? != 0 )) ; then
    echo "FAILURE insert ${VT}"
    exit 1
fi

echo $cmd "select * from t order by a"
$cmd "select * from t order by a" >> $OUT

echo "Waiting for ${VT} to rollout in 60 seconds"
sleep 60

timepart_stats 0

echo $cmd "select * from t order by a"
$cmd "select * from t order by a" >> $OUT

# TEST 4.
# create table with same name and check proper failure
header 4 "create table with same name and check proper failure"
echo $cmd "CREATE TABLE ${VT}(a int, b int)"
$cmd "CREATE TABLE ${VT}(a int, b int)"
if (( $? == 0 )) ; then
    echo "FAILURE dup table name"
    exit 1
fi

VT="t2"

# TEST 5.
# create old tpt together 
header 5 "create old tpt together"
echo $cmd "create table t3 (b int)"
$cmd "create table t3 (b int)"
if (( $? != 0 )) ; then
   echo "FAILURE create table t3"
   exit 1
fi
starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()-2*24*3600-60)'`
echo $cmd "CREATE TIME PARTITION ON t3 as ${VT} PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "CREATE TIME PARTITION ON t3 as ${VT} PERIOD 'daily' RETENTION 2 START '${starttime}'" 
if (( $? != 0 )) ; then
   echo "FAILURE create old stype partition"
   exit 1
fi

# wait for the rollouts to present 
sleep 30

# need to wait to avoid racing against shard drop event
timepart_stats 6

# TEST 6.
# create dup old time partition
header 6 "create dup old time partition"
echo $cmd "create table t4 (c int)"
$cmd "create table t4 (c int)"
if (( $? != 0 )) ; then
   echo "FAILURE create table t4"
   exit 1
fi
starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()-2*24*3600-60)'`
echo $cmd "CREATE TIME PARTITION ON t4 as ${VT} PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "CREATE TIME PARTITION ON t4 as ${VT} PERIOD 'daily' RETENTION 2 START '${starttime}'" 
if (( $? == 0 )) ; then
   echo "FAILURE dup old time partition"
   exit 1
fi

# 7.
# restart node, check partition read and events generation
header 7 "restart node, check partition read and events generation"
echo "Killing master now"
#kill_restart_node $master 1
#sleep 10

master=`$cmdt 'exec procedure sys.cmd.send("bdb cluster")' | grep MASTER | cut -f1 -d":" | tr -d '[:space:]'`
cmdm="cdb2sql ${CDB2_OPTIONS} --host $master $dbname default"
cmdmt="cdb2sql -tabs ${CDB2_OPTIONS} --host $master $dbname default"

timepart_stats 0

# 8.
# create table and alter to a partition; check inserts
header 8 "create table and alter to a partition; check inserts"
echo $cmd "create table t5 (d int)"
$cmd "create table t5 (d int)"
if (( $? != 0 )) ; then
   echo "FAILURE create table t5"
   exit 1
fi

$cmd "insert into t5 values (100), (200), (300)"
if (( $? != 0 )) ; then
   echo "FAILURE insert into table t5"
   exit 1
fi

starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()+15)'`
echo $cmd "ALTER TABLE t5 ADD COLUMN e int, PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "ALTER TABLE t5 ADD COLUMN e int, PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
if (( $? != 0 )) ; then
   echo "FAILURE to alter table t5 and add partion"
   exit 1
fi

timepart_stats 0

$cmd "insert into t5 values (101, 1), (201, 1), (301, 1)"
if (( $? != 0 )) ; then
   echo "FAILURE insert into table t5 round 2"
   exit 1
fi

sleep 15

$cmd "insert into t5 values (102, 2), (202, 2), (302, 2)"
if (( $? != 0 )) ; then
   echo "FAILURE insert into table t5 round 3"
   exit 1
fi

echo $cmd "select * from '\$0_65276E68' order by 1"
echo $cmd "select * from '\$0_65276E68' order by 1" >> $OUT
$cmd "select * from '\$0_65276E68' order by 1" >> $OUT
echo $cmd "select * from '\$1_DEE0E531' order by 1"
echo $cmd "select * from '\$1_DEE0E531' order by 1" >> $OUT
$cmd "select * from '\$1_DEE0E531' order by 1" >> $OUT

# 9.
# check attempt to partition an already partitioned table
header 9 "check attempt to partition an already partitioned table"
echo $cmd "ALTER TABLE t5 PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "ALTER TABLE t5 PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
if (( $? == 0 )) ; then
   echo "FAILURE to prevent duplicate partitioning"
   exit 1
fi

# 10. 
# alter (drop column) for  partitioned table (alias)
header 10 "alter (drop column) for  partitioned table (alias)"
echo $cmd "ALTER TABLE t5 DROP COLUMN e"
echo $cmd "ALTER TABLE t5 DROP COLUMN e" >> $OUT
$cmd "ALTER TABLE t5 DROP COLUMN e"
if (( $? != 0 )) ; then
   echo "FAILURE to alter table t5 and drop column e"
   exit 1
fi

echo $cmd "select * from t5 order by 1"
echo $cmd "select * from t5 order by 1" >> $OUT
$cmd "select * from t5 order by 1" >> $OUT

timepart_stats 0

# 11.
# Create and drop a partitioned table
header 11 "Create and drop a partitioned table"
starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()+20)'`
echo $cmd "CREATE TABLE t6 (a int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "CREATE TABLE t6 (a int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
if (( $? != 0 )) ; then
   echo "FAILURE to create partitioned table t6"
   exit 1
fi

timepart_stats 0

echo $cmd "DROP TABLE t6"
echo $cmd "DROP TABLE t6" >> $OUT
$cmd "DROP TABLE t6"
if (( $? != 0 )) ; then
   echo "FAILURE to drop table t6"
   exit 1
fi

timepart_stats 0

# 12.
# Test version across create/alter/drop
# Recreate a previous table, t5 partitioned by create, and drop it
# Recreate a t5 again unpartitioned, alter it to add partition, and drop it
header 12 "Test version across create/alter/drop"

echo $cmd "DROP TABLE t5"
echo $cmd "DROP TABLE t5" >> $OUT
$cmd "DROP TABLE t5"
if (( $? != 0 )) ; then
   echo "FAILURE to drop table t5"
   exit 1
fi

timepart_stats 0

starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()+120)'`
echo $cmd "CREATE TABLE t5 (a int, b int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "CREATE TABLE t5 (a int, b int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
if (( $? != 0 )) ; then
   echo "FAILURE to create partitioned table t5 after drop"
   exit 1
fi

timepart_stats 0

echo $cmd "DROP TABLE t5"
echo $cmd "DROP TABLE t5" >> $OUT
$cmd "DROP TABLE t5"
if (( $? != 0 )) ; then
   echo "FAILURE to drop table t5 after recreate"
   exit 1
fi

timepart_stats 0

echo $cmd "CREATE TABLE t5(c int)"
echo $cmd "CREATE TABLE t5(c int)" >> $OUT
$cmd "CREATE TABLE t5(c int)"
if (( $? != 0 )) ; then
   echo "FAILURE to recreate unpartitioned table t5"
   exit 1
fi

timepart_stats 0

echo $cmd "ALTER TABLE t5 PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
$cmd "ALTER TABLE t5 PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'" 
if (( $? != 0 )) ; then
   echo "FAILURE to alter-partition table t5"
   exit 1
fi

timepart_stats 0

echo $cmd "DROP TABLE t5"
echo $cmd "DROP TABLE t5" >> $OUT
$cmd "DROP TABLE t5"
if (( $? != 0 )) ; then
   echo "FAILURE to drop table t5 after alter-partition"
   exit 1
fi

timepart_stats 0

header 13 "Create table and merge existing table in it"
echo $cmd "CREATE TABLE t10(a int, b int)"
$cmd "CREATE TABLE t10(a int, b int)"
if (( $? != 0 )) ; then
   echo "FAILURE to create table t10"
   exit 1
fi

echo $cmd "insert into t10 values (1,2), (3,4), (5,6)"
$cmd "insert into t10 values (1,2), (3,4), (5,6)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into t10"
   exit 1
fi

echo $cmd "CREATE TABLE t11(a int) MERGE t10"
$cmd "CREATE TABLE t11(a int) MERGE t10"
if (( $? != 0 )) ; then
   echo "FAILURE to create t11 merge t10"
   exit 1
fi

echo $cmd "select * from t11 order by a"
$cmd "select * from t11 order by a" >> $OUT

header 14 "Create partitioned table, add records, drop partitiong and check table"
echo $cmd "CREATE TABLE t14(a int, b int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'"
$cmd "CREATE TABLE t14(a int, b int) PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START '${starttime}'"
if (( $? != 0 )) ; then
   echo "FAILURE to create partition t14"
   exit 1
fi

echo $cmd "insert into t14 values (10,20), (30,40), (50,60)"
$cmd "insert into t14 values (10,20), (30,40), (50,60)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into t14"
   exit 1
fi

echo $cmd "insert into '\$1_9ABA178A' values (111,111)"
$cmd "insert into '\$1_9ABA178A' values (111,111)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into t14i shard 1"
   exit 1
fi

echo $cmd "insert into '\$0_528953B1' values (222,222)"
$cmd "insert into '\$0_528953B1' values (222,222)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into t14i shard 0"
   exit 1
fi


echo $cmd "select * from t14 order by 1"
echo $cmd "select * from t14 order by 1" >> $OUT
$cmd "select * from t14 order by 1" >> $OUT

timepart_stats 0

$cmdm "exec procedure sys.cmd.send('convert_record_sleep 1')"

echo $cmd "ALTER TABLE t14 PARTITIONED BY NONE &"
$cmd "ALTER TABLE t14 PARTITIONED BY NONE" &
waitpid=$!

sleep 1

if $cmd "ALTER TABLE t14 PARTITIONED BY NONE"; then
   echo "FAILURE to block concurrent schema change"
   exit 1
fi

if ! wait $waitpid; then
   echo "FAILURE to remove partitioning for t14"
   exit 1
fi

$cmdm "exec procedure sys.cmd.send('convert_record_sleep 0')"

echo $cmd "select * from t14 order by 1"
echo $cmd "select * from t14 order by 1" >> $OUT
$cmd "select * from t14 order by 1" >> $OUT

timepart_stats 0

header 15 "Test ALTER table MERGE"
echo $cmd "CREATE TABLE t15(a int, b int)"
$cmd "CREATE TABLE t15(a int, b int)"
if (( $? != 0 )) ; then
   echo "FAILURE to create table t15"
   exit 1
fi
echo $cmd "CREATE TABLE t16(a int)"
$cmd "CREATE TABLE t16(a int)"
if (( $? != 0 )) ; then
   echo "FAILURE to create table t16"
   exit 1
fi

echo $cmd "insert into t15 values (11,12), (22,23), (33,34)"
$cmd "insert into t15 values (11,12), (22,23), (33,34)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into t15"
   exit 1
fi

echo $cmd "insert into t16 values (100), (200)"
$cmd "insert into t16 values (100), (200)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into t16"
   exit 1
fi

echo $cmd "alter table t15 drop column b, merge t16"
$cmd "alter table t15 drop column b, merge t16"
if (( $? != 0 )) ; then
   echo "FAILURE to alter merge"
   exit 1
fi

echo $cmd "select * from t15 order by a"
$cmd "select * from t15 order by a" >> $OUT

echo $cmd "select * from t16 order by a"
$cmd "select * from t16 order by a" >> $OUT 2>&1

timepart_stats 0

header 16 "Test PARTITION NONE on ALIASED table"
echo $cmd "CREATE TABLE t17(a int)"
$cmd "CREATE TABLE t17(a int)"
if (( $? != 0 )) ; then
   echo "FAILURE to create t17"
   exit 1
fi


echo $cmd "insert into t17 values (1000), (2000)"
$cmd "insert into t17 values (1000), (2000)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into t17"
   exit 1
fi

echo $cmd "ALTER TABLE t17 PARTITIONED BY TIME PERIOD 'daily' RETENTION 3 start '2023-01-01T'"
$cmd "ALTER TABLE t17 PARTITIONED BY TIME PERIOD 'daily' RETENTION 3 start '2023-01-01T'"
if (( $? != 0 )) ; then
   echo "FAILURE to partition t17"
   exit 1
fi

echo $cmd "insert into t17 values (3000), (4000)"
$cmd "insert into t17 values (3000), (4000)"
if (( $? != 0 )) ; then
   echo "FAILURE to insert into partition t17"
   exit 1
fi

timepart_stats 0

echo $cmd "ALTER TABLE t17 PARTITIONED BY NONE"
$cmd "ALTER TABLE t17 PARTITIONED BY NONE"
if (( $? != 0 )) ; then
   echo "FAILURE to UNpartition t17"
   exit 1
fi

echo $cmd "select * from t17 order by a"
$cmd "select * from t17 order by a" >> $OUT 2>&1


echo $cmd "select * from comdb2_tables where tablename='t17'"
$cmd "select * from comdb2_tables where tablename='t17'" >> $OUT 2>&1

echo $cmd "select * from comdb2_timepartshards where name='t17'"
$cmd "select * from comdb2_timepartshards where name='t17'" >> $OUT 2>&1

header 17 "Test block alter"
echo $cmd "create table t18(a int)"
$cmd "create table t18(a int)"
if (( $? != 0 )) ; then
   echo "FAILURE to create table t18"
   exit 1
fi

echo $cmd "put tunable disable_tagged_api_writes 0"
if [[ -n "$CLUSTER" ]]; then
    for node in $CLUSTER; do
        cdb2sql --host $node $dbname --host $node "put tunable disable_tagged_api_writes 0"
    done
else
    $cmd "put tunable disable_tagged_api_writes 0"
fi
if (( $? != 0 )) ; then
   echo "FAILURE to put tunable to 0"
   exit 1
fi

echo $cmd "ALTER TABLE t18 PARTITIONED BY TIME PERIOD 'daily' RETENTION 3 start '2024-01-01T'"
$cmd "ALTER TABLE t18 PARTITIONED BY TIME PERIOD 'daily' RETENTION 3 start '2024-01-01T'"
if (( $? != 0 )) ; then
   echo "FAILURE to block alter partition when tag writes are enabled"
   exit 1
fi

echo $cmd "put tunable disable_tagged_api_writes 1"
if [[ -n "$CLUSTER" ]]; then
    for node in $CLUSTER; do
        cdb2sql --host $node $dbname --host $node "put tunable disable_tagged_api_writes 1"
    done
else
    $cmd "put tunable disable_tagged_api_writes 1"
fi
if (( $? != 0 )) ; then
   echo "FAILURE to put tunable to 1"
   exit 1
fi

header 18 "Test check for partition new over old shards"
echo $cmd "create table t19(a int)"
$cmd "create table t19(a int)"
if (( $? != 0 )) ; then
   echo "FAILURE to create t19"
   exit 1
fi

starttime=`perl -MPOSIX -le 'local $ENV{TZ}=":/usr/share/zoneinfo/UTC"; print strftime "%Y-%m-%dT%H%M%S UTC", localtime(time()-3*24*3600-60)'`
echo $cmd "CREATE TIME PARTITION ON t19 as tv19 PERIOD 'daily' RETENTION 2 START '${starttime}'"
$cmd "CREATE TIME PARTITION ON t19 as tv19 PERIOD 'daily' RETENTION 2 START '${starttime}'"
if (( $? != 0 )) ; then
   echo "FAILURE to create time partition tv19"
   exit 1
fi

sleep 30

echo $cmd "drop time partition tv19"
$cmd "drop time partition tv19"
if (( $? != 0 )) ; then
   echo "FAILURE to drop time partition tv19"
   exit 1
fi

sleep 10

echo $cmd "create table t19(a int)"
$cmd "create table t19(a int)"
if (( $? != 0 )) ; then
   echo "FAILURE to recreate t19"
   exit 1
fi

echo $cmd "alter table t19 partitioned by time period 'daily' retention 2 start '20240901T'"
$cmd "alter table t19 partitioned by time period 'daily' retention 2 start '20240901T'"
if (( $? == 0 )) ; then
   echo "FAILURE to block creating overlapping shards"
   exit 1
fi

header 19 "Test block put retention"
echo $cmd "put time partition t17 retention 20"
$cmd "put time partition t17 retention 20" >> $OUT 2>&1

header 20 "Test block alter of a constraint target table"
$cmd "create table t21 {`cat t21.csc2`}"
if (( $? != 0 )) ; then
   echo "FAILURE to create table tv21"
   exit 1
fi


$cmd "create table t22 {`cat t22.csc2`}"
if (( $? != 0 )) ; then
   echo "FAILURE to create table tv22"
   exit 1
fi

$cmd "alter table t21 partitioned by time period 'daily' retention 2 start '2024-01-01T'"
if (( $? == 0 )) ; then
   echo "FAILURE to block partitioning a constraint target table"
   exit 1
fi



# we need to scrub dbname from alpha
sed "s/dorintdb/$dbname/g; s#\${CDB2_OPTIONS}#${CDB2_OPTIONS}#g" $OUT.alpha > $OUT.alpha.actual


difs=`diff $OUT $OUT.alpha.actual`
if [[ ! -z "${difs}" ]] ; then
   echo "diff $OUT $OUT.alpha.actual"
   echo ${difs}
   echo "FAILURE"
   exit 1
fi

echo "SUCCESS"