Current section
Files
Jump to
Current section
Files
lib/snmpkit/snmp_sim/test_helpers/stability_test_helper.ex
defmodule SnmpKit.SnmpSim.TestHelpers.StabilityTestHelper do
@moduledoc """
Specialized testing utilities for stability and endurance testing.
"""
alias SnmpKit.SnmpSim.{Device, LazyDevicePool}
@doc """
Monitors memory usage over a specified duration while running a test function.
"""
def monitor_memory_usage(duration_ms, sample_interval_ms, test_function) do
monitor_task =
Task.async(fn ->
monitor_memory_over_time(duration_ms, sample_interval_ms)
end)
test_task =
Task.async(fn ->
run_test_cycles(duration_ms, test_function)
end)
Task.await(test_task, :infinity)
memory_samples = Task.await(monitor_task, :infinity)
memory_samples
end
@doc """
Analyzes memory usage patterns to detect leaks and trends.
"""
def analyze_memory_samples(samples) when is_list(samples) do
initial_memory = List.first(samples)
final_memory = List.last(samples)
max_memory = Enum.max(samples)
avg_memory = Enum.sum(samples) / length(samples)
{initial_memory, final_memory, max_memory, avg_memory}
end
@doc """
Runs a comprehensive load test with monitoring.
"""
def run_load_test(devices, target_rps, duration_ms, options \\ %{}) do
monitor_response_times = Map.get(options, :monitor_response_times, false)
monitor_error_rates = Map.get(options, :monitor_error_rates, false)
monitor_resource_usage = Map.get(options, :monitor_resource_usage, false)
monitor_process_counts = Map.get(options, :monitor_process_counts, false)
# Start monitoring tasks
monitoring_tasks =
start_load_test_monitoring(
duration_ms,
monitor_response_times,
monitor_error_rates,
monitor_resource_usage,
monitor_process_counts
)
# Run the actual load test
load_test_task =
Task.async(fn ->
execute_load_test(devices, target_rps, duration_ms)
end)
# Await results
load_results = Task.await(load_test_task, :infinity)
monitoring_results = await_monitoring_tasks(monitoring_tasks)
# Combine results
Map.merge(load_results, monitoring_results)
end
@doc """
Runs an endurance test with varying workload patterns.
"""
def run_endurance_test(duration_ms, options \\ %{}) do
workload_patterns = Map.get(options, :workload_patterns, [])
inject_failures = Map.get(options, :inject_failures, false)
failure_rate = Map.get(options, :failure_rate, 0.0)
monitor_everything = Map.get(options, :monitor_everything, true)
start_time = System.monotonic_time(:millisecond)
end_time = start_time + duration_ms
# Initialize monitoring
monitoring_data =
if monitor_everything do
start_comprehensive_monitoring(duration_ms)
else
%{}
end
# Execute endurance test cycles
results =
execute_endurance_cycles(
workload_patterns,
start_time,
end_time,
inject_failures,
failure_rate,
%{
total_cycles: 0,
devices_created: 0,
requests_processed: 0,
errors_encountered: 0,
system_crashed: false,
memory_leak_detected: false,
deadlock_detected: false,
resource_exhaustion: false
}
)
# Finalize monitoring
final_monitoring =
if monitor_everything do
finalize_comprehensive_monitoring(monitoring_data)
else
%{}
end
Map.merge(results, final_monitoring)
end
@doc """
Checks overall system health and returns metrics.
"""
def check_system_health do
memory_info = :erlang.memory()
process_count = :erlang.system_info(:process_count)
# Check device pool health
device_stats =
try do
LazyDevicePool.get_stats()
catch
_type, _error -> %{active_count: 0, devices_created: 0}
end
# Calculate health metrics
# Convert to GB
memory_usage_ratio = memory_info[:total] / (1024 * 1024 * 1024)
%{
memory_usage: memory_usage_ratio,
memory_total_bytes: memory_info[:total],
process_count: process_count,
active_devices: device_stats.active_count,
devices_created: Map.get(device_stats, :devices_created, 0),
system_healthy: memory_usage_ratio < 2.0 and process_count < 50_000
}
end
@doc """
Injects various types of failures for testing system resilience.
"""
def inject_failure(scenario) do
case scenario.action do
:kill_random_processes ->
kill_random_processes(scenario.count)
:memory_pressure ->
create_memory_pressure(scenario.intensity)
:exhaust_ports ->
exhaust_ports(scenario.percentage)
:inject_network_delays ->
inject_network_delays(scenario.delay_ms)
:cascading_failures ->
trigger_cascading_failures(scenario.failure_rate)
end
end
@doc """
Clears any active failure injections.
"""
def clear_failure_injection do
# Clear network delay injection
Process.whereis(:network_delay_injector) |> stop_process_if_exists()
# Clear memory pressure
Process.whereis(:memory_pressure_creator) |> stop_process_if_exists()
# Clear port exhaustion
Process.whereis(:port_exhauster) |> stop_process_if_exists()
:ok
end
@doc """
Resets system state for stability testing.
"""
def reset_system_state do
# Force garbage collection
:erlang.garbage_collect()
# Clear device pool caches
try do
LazyDevicePool.clear_cache()
catch
_type, _error -> :ok
end
# Reset any test state
clear_failure_injection()
:ok
end
@doc """
Cleanup all test resources.
"""
def cleanup_all do
reset_system_state()
# Kill any remaining test processes
test_processes =
Process.registered()
|> Enum.filter(fn name -> String.starts_with?(Atom.to_string(name), "test_") end)
Enum.each(test_processes, fn name ->
Process.whereis(name) |> stop_process_if_exists()
end)
:ok
end
# Private helper functions
defp monitor_memory_over_time(duration_ms, sample_interval_ms) do
end_time = System.monotonic_time(:millisecond) + duration_ms
collect_memory_samples(end_time, sample_interval_ms, [])
end
defp collect_memory_samples(end_time, interval_ms, samples) do
current_time = System.monotonic_time(:millisecond)
if current_time >= end_time do
Enum.reverse(samples)
else
memory_info = :erlang.memory()
new_samples = [memory_info[:total] | samples]
Process.sleep(interval_ms)
collect_memory_samples(end_time, interval_ms, new_samples)
end
end
defp run_test_cycles(duration_ms, test_function) do
end_time = System.monotonic_time(:millisecond) + duration_ms
execute_test_cycles(end_time, test_function, 0)
end
defp execute_test_cycles(end_time, test_function, cycle_count) do
current_time = System.monotonic_time(:millisecond)
if current_time >= end_time do
cycle_count
else
test_function.()
execute_test_cycles(end_time, test_function, cycle_count + 1)
end
end
defp start_load_test_monitoring(
duration_ms,
monitor_response_times,
monitor_error_rates,
monitor_resource_usage,
monitor_process_counts
) do
tasks = []
tasks =
if monitor_response_times do
[Task.async(fn -> monitor_response_times(duration_ms) end) | tasks]
else
tasks
end
tasks =
if monitor_error_rates do
[Task.async(fn -> monitor_error_rates(duration_ms) end) | tasks]
else
tasks
end
tasks =
if monitor_resource_usage do
[Task.async(fn -> monitor_resource_usage(duration_ms) end) | tasks]
else
tasks
end
tasks =
if monitor_process_counts do
[Task.async(fn -> monitor_process_counts(duration_ms) end) | tasks]
else
tasks
end
tasks
end
defp await_monitoring_tasks(tasks) do
results = Task.await_many(tasks, :infinity)
# Combine monitoring results
%{
response_times: Enum.at(results, 0, []),
error_rates: Enum.at(results, 1, []),
resource_usage: Enum.at(results, 2, []),
process_counts: Enum.at(results, 3, [])
}
end
defp execute_load_test(devices, target_rps, duration_ms) do
request_interval_ms = 1000 / target_rps
end_time = System.monotonic_time(:millisecond) + duration_ms
execute_load_requests(devices, request_interval_ms, end_time, 0, 0)
end
defp execute_load_requests(devices, interval_ms, end_time, total_requests, errors) do
current_time = System.monotonic_time(:millisecond)
if current_time >= end_time do
%{
total_requests: total_requests,
errors: errors,
actual_rps:
total_requests / ((end_time - (end_time - System.monotonic_time(:millisecond))) / 1000)
}
else
device = Enum.random(devices)
result =
try do
Device.get(device, "1.3.6.1.2.1.1.1.0")
catch
_type, _error -> {:error, :exception}
end
new_errors =
case result do
{:ok, _} -> errors
_ -> errors + 1
end
Process.sleep(round(interval_ms))
execute_load_requests(devices, interval_ms, end_time, total_requests + 1, new_errors)
end
end
defp start_comprehensive_monitoring(duration_ms) do
memory_task = Task.async(fn -> monitor_memory_over_time(duration_ms, 1000) end)
process_task = Task.async(fn -> monitor_process_counts(duration_ms) end)
%{
memory_task: memory_task,
process_task: process_task,
start_time: System.monotonic_time(:millisecond)
}
end
defp finalize_comprehensive_monitoring(monitoring_data) do
memory_samples = Task.await(monitoring_data.memory_task, :infinity)
process_samples = Task.await(monitoring_data.process_task, :infinity)
%{
memory_samples: memory_samples,
process_samples: process_samples
}
end
defp execute_endurance_cycles(
patterns,
start_time,
end_time,
inject_failures,
failure_rate,
results
) do
current_time = System.monotonic_time(:millisecond)
if current_time >= end_time do
results
else
# Execute workload pattern
pattern = Enum.random(patterns)
cycle_results = execute_workload_pattern(pattern)
# Inject failures if enabled
if inject_failures and :rand.uniform() < failure_rate do
inject_random_failure()
end
# Update results
updated_results = %{
results
| total_cycles: results.total_cycles + 1,
devices_created: results.devices_created + cycle_results.devices_created,
requests_processed: results.requests_processed + cycle_results.requests_processed,
errors_encountered: results.errors_encountered + cycle_results.errors_encountered
}
# Check for system issues
health = check_system_health()
system_issues = %{
system_crashed: !health.system_healthy,
memory_leak_detected: detect_memory_leak(health),
deadlock_detected: detect_deadlock(),
resource_exhaustion: detect_resource_exhaustion(health)
}
final_results = Map.merge(updated_results, system_issues)
# Continue or stop based on system health
if final_results.system_crashed do
final_results
else
# Brief pause between cycles
Process.sleep(1000)
execute_endurance_cycles(
patterns,
start_time,
end_time,
inject_failures,
failure_rate,
final_results
)
end
end
end
defp execute_workload_pattern(pattern) do
case pattern.type do
:steady ->
execute_steady_workload(pattern.device_count, pattern.duration_minutes)
:burst ->
execute_burst_workload(pattern.device_count, pattern.duration_minutes)
:idle ->
execute_idle_workload(pattern.device_count, pattern.duration_minutes)
end
end
defp execute_steady_workload(device_count, _duration_minutes) do
# Implement steady workload pattern
%{devices_created: device_count, requests_processed: device_count * 10, errors_encountered: 0}
end
defp execute_burst_workload(device_count, _duration_minutes) do
# Implement burst workload pattern
%{devices_created: device_count, requests_processed: device_count * 20, errors_encountered: 1}
end
defp execute_idle_workload(device_count, _duration_minutes) do
# Implement idle workload pattern - simulate low activity instead of literal waiting
# Brief pause to simulate idle state without extending test duration
Process.sleep(100)
%{devices_created: device_count, requests_processed: device_count * 2, errors_encountered: 0}
end
defp inject_random_failure do
failures = [:network_delay, :memory_pressure, :process_kill]
failure = Enum.random(failures)
case failure do
:network_delay -> inject_network_delays(100)
:memory_pressure -> create_memory_pressure(:low)
:process_kill -> kill_random_processes(1)
end
end
defp kill_random_processes(count) do
# Get only device processes, not critical system processes
all_device_processes =
Process.list()
|> Enum.filter(fn pid ->
is_device_process?(pid) and Process.alive?(pid)
end)
device_processes =
Enum.take_random(all_device_processes, min(count, length(all_device_processes)))
require Logger
Logger.info("Killing #{length(device_processes)} device processes for stability test")
Enum.each(device_processes, fn pid ->
try do
Process.exit(pid, :kill)
catch
_type, _error -> :ok
end
end)
end
# Helper function to identify if a process is a device process (safe to kill)
defp is_device_process?(pid) do
try do
case Process.info(pid, :registered_name) do
{:registered_name, []} ->
# Check if it's a device process by looking at initial call
case Process.info(pid, :initial_call) do
{:initial_call, {SnmpKit.SnmpSim.Device, :init, 1}} -> true
{:initial_call, {SnmpKit.SnmpSim.Core.Server, :init, 1}} -> true
_ -> false
end
{:registered_name, name} ->
# Don't kill critical named processes
case name do
SnmpKit.SnmpSim.MIB.SharedProfiles -> false
SnmpKit.SnmpSim.LazyDevicePool -> false
SnmpKit.SnmpSim.Application -> false
_ -> false
end
nil ->
false
end
catch
_type, _error -> false
end
end
defp create_memory_pressure(intensity) do
size =
case intensity do
# 10MB
:low -> 10_000_000
# 50MB
:medium -> 50_000_000
# 100MB
:high -> 100_000_000
end
spawn(fn ->
# Create memory pressure by allocating large binaries
_large_binary = :binary.copy(<<0>>, size)
# Hold memory for 10 seconds
Process.sleep(10_000)
end)
end
defp exhaust_ports(percentage) when percentage > 0 and percentage <= 1 do
max_ports = :erlang.system_info(:port_limit)
target_ports = round(max_ports * percentage)
spawn(fn ->
# Open many ports to exhaust the limit
_ports =
Enum.map(1..target_ports, fn _i ->
case :gen_udp.open(0) do
{:ok, socket} -> socket
_ -> nil
end
end)
# Hold ports for 30 seconds
Process.sleep(30_000)
end)
end
defp inject_network_delays(_delay_ms) do
# This is a simulation - in a real implementation you might
# intercept network calls or use a proxy
spawn(fn ->
Process.register(self(), :network_delay_injector)
receive do
:stop -> :ok
after
# Auto-stop after 1 minute
60_000 -> :ok
end
end)
end
defp trigger_cascading_failures(failure_rate) do
# Simulate cascading failures by randomly killing device processes
device_processes =
Process.registered()
|> Enum.filter(fn name -> String.contains?(Atom.to_string(name), "device") end)
failure_count = round(length(device_processes) * failure_rate)
Enum.take_random(device_processes, failure_count)
|> Enum.each(fn process_name ->
pid = Process.whereis(process_name)
if pid && Process.alive?(pid) do
Process.exit(pid, :kill)
end
end)
end
defp monitor_response_times(_duration_ms) do
# Implementation for monitoring response times
# Placeholder
[]
end
defp monitor_error_rates(_duration_ms) do
# Implementation for monitoring error rates
# Placeholder
[]
end
defp monitor_resource_usage(_duration_ms) do
# Implementation for monitoring resource usage
# Placeholder
[]
end
defp monitor_process_counts(duration_ms) do
end_time = System.monotonic_time(:millisecond) + duration_ms
collect_process_samples(end_time, [])
end
defp collect_process_samples(end_time, samples) do
current_time = System.monotonic_time(:millisecond)
if current_time >= end_time do
Enum.reverse(samples)
else
process_count = :erlang.system_info(:process_count)
new_samples = [process_count | samples]
Process.sleep(1000)
collect_process_samples(end_time, new_samples)
end
end
defp detect_memory_leak(health) do
# Simple heuristic: memory usage > 3GB suggests potential leak
health.memory_usage > 3.0
end
defp detect_deadlock do
# Simple deadlock detection based on process count growth
current_count = :erlang.system_info(:process_count)
current_count > 75_000
end
defp detect_resource_exhaustion(health) do
health.memory_usage > 3.5 or health.process_count > 90_000
end
defp stop_process_if_exists(nil), do: :ok
defp stop_process_if_exists(pid) when is_pid(pid) do
if Process.alive?(pid) do
Process.exit(pid, :normal)
end
end
end