from pdstools import IH

# from pdstools.utils import cdh_utils

import polars as pl
import plotly as plotly


# plotly.offline.init_notebook_mode()
# pio.renderers.default = "vscode"

Example IH Analysis

Interaction History (IH) is a rich source of data at the level of individual interactions from Pega DSM applications like CDH. It contains the time of the interaction, the channel, the actions/treatments, the customer ID and is used to track different types of outcomes (decisions, sends, opens, clicks, etc). It does not contain details of individual customers - only their ID’s.

Interaction History is typically used to analyze customer behavior and optimize decision strategies. The following sections provide various example analyses that can be performed on IH data, including distribution analysis, response analysis, success rates, model performance, propensity distribution, and response time analysis.

Like most of PDSTools, it uses plotly for visualization and polars (dataframe) but the purpose of this Notebook is more to serve example analyses than re-usable code, although of course we do try to provide some generic, re-usable functions. All of the analyses should be able to be replicated easily in other analytical BI environments - except perhaps the analysis of model performance / AUC.

This notebook uses sample data shipped with PDStools. Replace it with your own actual IH data and modify the analyses as appropriate.

# ih = IH.from_ds_export(
#     "../../data/Data-pxStrategyResult_pxInteractionHistory_20210101T010000_GMT.zip"
# )
ih = IH.from_mock_data(n=1e5)

Preview of the raw IH data

ih.data.head().collect()

shape: (5, 13)

InteractionID	Channel	Issue	Group	Name	Treatment	ExperimentGroup	ModelTechnique	OutcomeTime	Direction	Outcome	BasePropensity	Propensity
str	str	str	str	str	str	str	str	datetime[μs]	str	str	f64	f64
"1000000000"	"Email"	"Retention"	"Lending"	"Lending_4"	"Lending_4_EmailTreatment2"	"Conversion-Test"	"NaiveBayes"	2025-01-13 16:30:22.456470	"Outbound"	"Pending"	0.003544	0.002902
"1000000001"	"Web"	"Service"	"Savings"	"Savings_8"	"Savings_8_WebTreatment2"	"Conversion-Test"	"GradientBoost"	2025-01-13 16:29:04.696470	"Inbound"	"Impression"	0.008536	0.007802
"1000000002"	"Email"	"Retention"	"Insurance"	"Insurance_2"	"Insurance_2_EmailTreatment2"	"Conversion-Control"	"NaiveBayes"	2025-01-13 16:27:46.936470	"Outbound"	"Pending"	0.020755	0.021987
"1000000003"	"Email"	"Retention"	"Savings"	"Savings_6"	"Savings_6_EmailTreatment2"	"Conversion-Control"	"GradientBoost"	2025-01-13 16:26:29.176470	"Outbound"	"Pending"	0.006918	0.007077
"1000000004"	"Email"	"Service"	"Pension"	"Pension_1"	"Pension_1_EmailTreatment1"	"Conversion-Test"	"NaiveBayes"	2025-01-13 16:25:11.416470	"Outbound"	"Pending"	0.004687	0.004359

The same interaction can occur multiple times: once when the first decision is made, then later when responses are captured (accepted, sent, clicked, etc.). For some of the analyses we need to group by interaction. This is how that data looks like:

ih.aggregates._summary_interactions(by=["Channel"]).head().collect()

shape: (5, 7)

Channel	InteractionID	Interaction_Outcome_Engagement	Interaction_Outcome_Conversion	Interaction_Outcome_OpenRate	Propensity	Outcomes
str	str	bool	bool	bool	f64	list[str]
"Email"	"1000006714"	false	false	false	0.025317	["Pending"]
"Email"	"1000096180"	false	false	false	0.003265	["Pending"]
"Email"	"1000010011"	false	false	false	0.003287	["Pending"]
"Email"	"1000028971"	false	false	false	0.008994	["Pending"]
"Web"	"1000001294"	false	false	false	0.037135	["Impression"]

Distribution Analysis

A distribution of the offers (actions/treatments) is often the most obvious type of analysis. You can do an action distribution for specific outcomes (what is offered, what is accepted), view it conditionally (what got offered last month vs this month) - possibly with a delta view, or over time.

ih.plot.response_count_tree_map()

fig = ih.plot.action_distribution(
    query=pl.col.Outcome.is_in(["Clicked", "Accepted"]),
    title="Distribution of Actions",
    color="Outcome",
)
# fig.update_layout(yaxis=dict(tickmode="linear")) # to show all names
fig

Response Analysis

A simple view of the responses over time shows how many responses are received per day (or any other period).

ih.plot.response_count(every="1d")

Which could be viewed per channel as well:

ih.plot.response_count(
    facet="Channel",
    query=pl.col.Channel != "",
)

Success Rates

Success rates (accept rate, open rate, conversion rate) are interesting to track over time. In addition you may want to split by e.g. Channel, or contrast the rates for different experimental setups in an A-B testing set-up.

ih.plot.success_rate(
    facet="Channel", query=pl.col.Channel.is_not_null() & (pl.col.Channel != "")
)

Model Performance

Similar to Success Rates: typically viewed over time, likely split by channel, conditioned on variations, e.g. NB vs AGB models.

ih.plot.model_performance_trend(by="Channel", every="1w")

AGB vs NB analysis

There are different types of ADM models you can use in CDH. This analysis shows the model performance of the (classic) Naive Bayes models vs the new Gradient Boosting models. We split by channel as this often matters.

fig = ih.plot.model_performance_trend(
    by="ModelTechnique",
    facet="Channel",
    every="1w",
    title="Model Performance of Naive Bayes vs Gradient Boosting Models",
)
fig.update_layout(legend_title_text="Technique")
fig

Propensity Distribution

IH also contains information about the factors that determine the prioritization of the offers: lever values, propensities etc.

Here we show the distribution of the propensities of the offers made. It’s also a first example of a custom analysis not currently supported directly by the PDSTools library. You can see how we access the underlying IH data (ih.data), then aggregate and display it.

import plotly.figure_factory as ff

channels = [
    c
    for c in ih.data.select(pl.col.Channel.unique().sort())
    .collect()["Channel"]
    .to_list()
    if c is not None and c != ""
]

plot_data = [
    ih.data.filter(pl.col.Channel == c)
    .select(["Propensity"])
    .collect()["Propensity"]
    .sample(fraction=0.1)
    .to_list()
    for c in channels
]
fig = ff.create_distplot(plot_data, group_labels=channels, show_hist=False)
fig.update_layout(
    title="Propensity Distribution",
    yaxis=dict(showticklabels=False),
    xaxis=dict(title="Propensity", tickformat=".0%"),
    legend_title_text="Channel",
    template="pega",
)
fig

Response Time Analysis

Time is one of the dimensions in IH. Here we take a look at how subsequent responses relate to the original decision. It shows, for example, how much time there typically is between the moment of decision and the click.

This type of analysis is usually part of attribution analysis when considering conversion modeling.

import plotly.express as px

outcomes = [
    objective
    for objective in ih.data.select(pl.col.Outcome.unique().sort())
    .collect()["Outcome"]
    .to_list()
    if objective is not None and objective != ""
]
plot_data = (
    ih.data.filter(pl.col.OutcomeTime.is_not_null())
    .group_by("InteractionID")
    .agg(
        [pl.col.OutcomeTime.min().alias("Decision_Time")]
        + [
            pl.col.OutcomeTime.filter(pl.col.Outcome == o).max().alias(o)
            for o in outcomes
        ],
    )
    .collect()
    .unpivot(
        index=["InteractionID", "Decision_Time"],
        variable_name="Outcome",
        value_name="Time",
    )
    .with_columns(Duration=(pl.col.Time - pl.col.Decision_Time).dt.total_seconds())
    .filter(pl.col.Duration > 0)
)

ordered_outcomes = (
    plot_data.group_by("Outcome")
    .agg(Duration=pl.col("Duration").median())
    .sort("Duration")["Outcome"]
    .to_list()
)

fig = px.box(
    plot_data,
    x="Duration",
    y="Outcome",
    color="Outcome",
    template="pega",
    category_orders={"Outcome": ordered_outcomes},
    points=False,
    title="Duration of Responses",
    log_x=True,
)
fig.update_layout(
    xaxis_title="Duration (seconds) with logarithmic scale", yaxis_title=""
)
fig