source $srcdir/$subdir/onthefly_common.tcl

set test "kprobes_onthefly"
if {![installtest_p] || ![kprobes_disabling_p]} {
   untested "$test"
   return
}

# Goal: Ensure that kprobes properly support on-the-fly
# arming/disarming. The script consists of a vfs_read() probe and a
# vfs_read() return probe whose conditions are toggled by a timer
# probe. We ensure that the kprobes will always be triggered by
# running 'echo' in the background. We then analyze the actual debug
# statements printed out to ensure the toggled probes are armed and
# disarmed as expected.
#
# PR17831: Note that originally this script probed 'vfs.read' and
# 'vfs.read.return', but this picked up inlined versions of vfs_read()
# on some kernels (like 3.10.0-229.el7.ppc64), which made the output
# different than we expected. So, we've switched to probing
# 'kernel.function("vfs_read").call' and
# 'kernel.function("vfs_read").return' which should return consistent
# results.

# Runs stap on a script with the given arguments
proc run_stap {subtest args script} {
   global test

   # Start the background echos
   set echo_loop "while \[ true \]; do /bin/echo 1>/dev/null; done"
   set echo_pid [exec sh -c $echo_loop &]

   # We always need the DEBUG flag, and -u to avoid probe elision
   set args "$args -DDEBUG_STP_ON_THE_FLY -u"

   # The script we use uses @entry(), which uses maps underneath. Since we will
   # be stressing the system, we need to ensure we will have enough space for
   # many map items.
   set args "$args -DMAXMAPENTRIES=100000"

   # This test seems especially prone to exceeding the default overhead
   # threshold in the harder subtests. Disable overload checking.
   set args "$args -DSTP_NO_OVERLOAD"

   # Run stap and catch any errors (we will rethrow it later)
   set err [catch {eval exec stap -e {$script} $args} out]

   # Kill echo
   kill -INT $echo_pid 1

   if {$err} {error $out}
   return $out
}

# Returns a script with some variables replaced.
#     SUBTEST    name of subtest
#     ENABLED    set to 1 to start the target probe as enabled, 0 as disabled
#     MAXTOGGLES number of times we toggle the target probe condition
#     TIMER      if a number, then it's the period of the timer probe which
#                toggles the target probes, if set to 'profile' then instead a
#                timer.profile probe is used to toggle the target probe, if set
#                to 'hardcore' then both a timer.profile probe and a
#                kernel.trace("*") probe is used for toggling, if set to 'max'
#                then a lot of various probe points are used for toggling
proc make_script {SUBTEST ENABLED MAXTOGGLES TIMER} {
   global test

   set script {

      # We want these probes to fire as deterministically as possible so that
      # their outputs can easily be predicted and compared. Unfortunately this
      # is complicated by multiple facts, including
      #     (1) the timer might be so fast that the probes don't have time to
      #         fire and print something,
      #     (2) the kprobe might print something after it is re-enabled, but
      #         before the kretprobe is re-enabled,
      #     (3) disabling a kretprobe won't stop an already running handler.
      #
      # To get around these issues, we use a simple state machine. The states
      # are as follow:
      #
      #     0 = cond disabled
      #     1 = cond enabled, but kprobe && kretprobe not yet enabled
      #     2 = cond enabled, kprobe enabled, kretprobe not yet enabled
      #     3 = cond enabled, kprobe && kretprobe enabled, nothing printed yet
      #     4 = cond enabled, 'hit' printed but not 'rethit'
      #     5 = cond enabled, 'rethit' printed

      global state = ENABLED
      global toggles = 0

      probe kernel.function("vfs_read").call if (state > 0) {
         if (state == 3) {
            println("hit")
            state++;
         } else if (state == 1) {
            state++;
         }
      }

      probe kernel.function("vfs_read").return if (state > 0) {
         # ensure that nothing changed during the vfs_read body
         if (state != @entry(state))
            next
         if (state == 4) {
            println("rethit")
            state++;
         } else if (state == 2) {
            state++;
         }
      }

      probe timer.us(TIMER) {
         if (state != 0 && state != 5)
            next # give probes more time to move through the states
         toggles++
         if (toggles > MAXTOGGLES)
            exit()
         else {
            println("toggling")
            state = !state
         }
      }

      probe timer.s(360) {
         println("timed out")
         exit()
      }
   }

   # On ppc64, you can't probe the hcall_* tracepoints (PR17126:
   # tracepoints.exp testcase causing stalls/hang on ppc64). This was
   # worked around by making a tracepoint blocklist. However, this
   # test is using the '-g' (guru) argument to systemtap, which causes
   # systemtap to ignore the tracepoint blocklist. So, on ppc64, we'll
   # probe all the kernel tracepoints *except* for the ones starting
   # with 'h'.
   #
   # XXX: Also skip lock:lock_* tracepoints due to issues on recent rawhide.
   # TODOXXX: Recent ppc64 also has issues with one of the tracepoints under
   # 'm'. Needs further investigation.
   if {[string match ppc* $::tcl_platform(machine)]} {
      set kernel_trace {kernel.trace("[a-gi-kn-z]*")}
   } else {
      set kernel_trace {kernel.trace("[a-km-z]*")}
   }

   # Set in values
   set script [string map "TEST $test \
                           SUBTEST $SUBTEST \
                           ENABLED $ENABLED \
                           MAXTOGGLES $MAXTOGGLES" $script]
   if {[string match $TIMER "profile"]} {
      set script [string map "timer.us(TIMER) timer.profile" $script]
   } elseif {[string match $TIMER "hardcore"]} {
      set script [string map "timer.us(TIMER) \
         {timer.profile, $kernel_trace}" $script]
   } elseif {[string match $TIMER "max"]} {
      set max_pps "begin, end, error, kernel.function(\"*@workqueue.c\"), \
                   process.begin, process.end, $kernel_trace"
      if {[uprobes_p]} {
         set max_pps "$max_pps, process(\"echo\").function(\"*\")"
      }
      set max_pps "$max_pps, \
          netfilter.pf(\"NFPROTO_IPV4\").hook(\"NF_INET_LOCAL_IN\")?, \
          netfilter.pf(\"NFPROTO_IPV4\").hook(\"NF_INET_LOCAL_OUT\")?, \
          perf.sw.cpu_clock.sample(1000000)?, timer.profile.tick?"
      set script [string map "timer.us(TIMER) {$max_pps}" $script]
   } else {
      set script [string map "TIMER $TIMER" $script]
   }

   return $script
}

# Creates the pattern which will be used by the is_valid_output proc to
# determine if the output is valid. In other words, given the initial value of
# enabled, the maximum number of toggles, and whether on-the-fly is turned on or
# not, this proc determines what the output of stap will be.
proc make_pattern {subtest start_enabled maxtoggles} {
   set pattern ""

   # If we're not starting as enabled, then the kprobes will be registered as
   # disabled.
   if {!$start_enabled} {
      lappend pattern "* registering as disabled (k*probe) pidx ?"
      lappend pattern "* registering as disabled (k*probe) pidx ?"
      lappend pattern "* registering as disabled (k*probe) pidx ?"
   }

   # For each toggle:
   #  - if we were enabled, then we would have had 'hit' and 'rethit' lines
   #  - if i == maxtoggles, then we'll exit so nothing more will be printed
   #  - if i < maxtoggles, then the timer probe will output 'toggling' followed
   #    by 'disabling' or 'enabling' lines if on-the-fly is turned on
   for {set i 0} {$i <= $maxtoggles} {incr i} {
      if {$start_enabled} {
         lappend pattern "hit" "rethit"
      }
      if {$i < $maxtoggles} {
         lappend pattern "toggling"
         if {$start_enabled} {
            set start_enabled 0
            lappend pattern "* disabling (k*probe) pidx ?"
            lappend pattern "* disabling (k*probe) pidx ?"
            lappend pattern "* disabling (k*probe) pidx ?"
         } else {
            set start_enabled 1
            if {$i < $maxtoggles} {
               lappend pattern "* enabling (k*probe) pidx ?"
               lappend pattern "* enabling (k*probe) pidx ?"
               lappend pattern "* enabling (k*probe) pidx ?"
            }
         }
      }
   }

   return $pattern
}

# Various on-the-fly tests

# Toggle 0, 1, 2, 3, 4, and 5 times. This is important since we test if the
# probes finish well when disabled/enabled.

run_subtest_valid "otf_finish_at_start_disabled" 0 0 500000
run_subtest_valid "otf_finish_at_start_enabled"  1 0 500000
run_subtest_valid "otf_start_disabled_iter_1"    0 1 500000
run_subtest_valid "otf_start_enabled_iter_1"     1 1 500000
run_subtest_valid "otf_start_disabled_iter_2"    0 2 500000
run_subtest_valid "otf_start_enabled_iter_2"     1 2 500000
run_subtest_valid "otf_start_disabled_iter_3"    0 3 500000
run_subtest_valid "otf_start_enabled_iter_3"     1 3 500000
run_subtest_valid "otf_start_disabled_iter_4"    0 4 500000
run_subtest_valid "otf_start_enabled_iter_4"     1 4 500000
run_subtest_valid "otf_start_disabled_iter_5"    0 5 500000
run_subtest_valid "otf_start_enabled_iter_5"     1 5 500000

# Small interval tests

run_subtest_valid "otf_timer_100ms" 1 5 100000 \
   -D STP_ON_THE_FLY_INTERVAL=1000000
run_subtest_valid "otf_timer_50ms"  1 5 50000 \
   -D STP_ON_THE_FLY_INTERVAL=1000000
run_subtest_valid "otf_timer_10ms"  1 5 10000 \
   -D STP_ON_THE_FLY_INTERVAL=1000000

# NB: if we go any shorter, the kernel might not have a chance to run
# systemtap_module_refresh() before the next toggling occurs, throwing off the
# expected output.

# Stress tests

# These tests are not validated, i.e. their outputs are not checked against a
# pattern (see NB above why). Instead, they're just meant to stress the
# on-the-fly mechanism to ensure stap/the kernel doesn't die. So if stap is
# successfully run and terminated, we just default to a pass.

run_subtest_stress "otf_stress_5ms_iter_50"   1 50 5000 \
   -D STP_ON_THE_FLY_INTERVAL=100000
run_subtest_stress "otf_stress_1ms_iter_50"   1 50 1000 \
   -D STP_ON_THE_FLY_INTERVAL=100000
run_subtest_stress "otf_stress_500us_iter_50" 1 50 500 \
   -D STP_ON_THE_FLY_INTERVAL=100000
run_subtest_stress "otf_stress_100us_iter_50" 1 50 100 \
   -D STP_ON_THE_FLY_INTERVAL=100000

run_subtest_stress "otf_stress_prof_iter_2000" 1 2000 profile \
   -D STP_ON_THE_FLY_INTERVAL=100000

run_subtest_stress "otf_stress_hard_iter_2000" 1 2000 hardcore \
   -D STP_ON_THE_FLY_INTERVAL=100000

run_subtest_stress "otf_stress_max_iter_5000" 1 5000 max \
   -D STP_ON_THE_FLY_INTERVAL=100000
