Missing Values: Line, Path, Area, and Ribbon

Missing value (Double.NaN or null) handling in geomLine(), geomPath(), geomArea(), and geomRibbon().

Note: geomPath() handles missing values slightly differently than the others.

%useLatestDescriptors
%use dataframe
%use lets-plot

LetsPlot.getInfo()

Lets-Plot Kotlin API v.4.11.3-SNAPSHOT. Frontend: Notebook with dynamically loaded JS. Lets-Plot JS v.4.8.1rc1.
Outputs: Web (HTML+JS), Kotlin Notebook (Swing), Static SVG (hidden)

import java.time.format.DateTimeFormatter
import java.time.LocalDate

val economics_url = "https://raw.githubusercontent.com/JetBrains/lets-plot-docs/master/data/economics.csv"
val df = DataFrame.readCSV(economics_url)
val dfWithDate = df.convert("date") { LocalDate.parse(it.toString()) }

val dfFiltered = dfWithDate.filter { it["date"] as LocalDate >= LocalDate.of(2006, 1, 1) }
val economics = dfFiltered.toMap()

dfFiltered.head()

DataFrame: rowsCount = 5, columnsCount = 7

untitled	date	pce	pop	psavert	uempmed	unemploy
463	2006-01-01	9059.800000	297647.000000	4.200000	8.600000	7064
464	2006-02-01	9090.100000	297854.000000	4.200000	9.100000	7184
465	2006-03-01	9122.100000	298060.000000	4.200000	8.700000	7072
466	2006-04-01	9174.800000	298281.000000	4.000000	8.400000	7120
467	2006-05-01	9215.100000	298496.000000	3.800000	8.500000	6980

// Make some gaps in the data

val MILLISECONDS_IN_DAY = 24.0 * 60 * 60 * 1_000

val dfStep1 = dfFiltered.convert { "unemploy"<Int>() }.to<Double>()

val dfStep2 = dfStep1.update { "unemploy"<Double>() }
    .where { "date"<LocalDate>() in LocalDate.of(2012, 1, 1)..LocalDate.of(2012, 4, 1) }
    .with { Double.NaN }

val dfStep3 = dfStep2.update {it["date"]}
    .where {it as LocalDate in LocalDate.of(2008, 1, 1)..LocalDate.of(2009, 6, 1)}
    .withNull()

val economicsWithGaps = dfStep3.toMap()

fun epochMillis(y: Int, m: Int, d: Int): Double =
    LocalDate(y, m, d).toEpochDays() * MILLISECONDS_IN_DAY

val basePlot = letsPlot { x = "date"; y = "unemploy" } +
    geomLine(
        data = economics,
        size = 10.0, alpha = 0.1, tooltips = tooltipsNone
    ) +
        geomLabel(
            label = "Missing dates",
            x = epochMillis(2008, 8, 15), y = 11000.0,
            nudgeX = -70.0, nudgeUnit = "px"
        ) +
        geomLabel(
            label = "Missing unemployment\nfigures",
            x = epochMillis(2012, 2, 15), y = 13000.0,
            nudgeX = 80.0, nudgeY = 40.0, nudgeUnit = "px"
        ) +
    themeClassic() +
    ggsize(800, 300)

1. Line

basePlot +
    geomLine(
        data = economicsWithGaps,
        color = "teal"
    )

2. Path

basePlot +
    geomPath(
        data = economicsWithGaps,
        color = "teal"
    )

3. Area

basePlot +
    geomArea(
        data = economicsWithGaps,
        color = "teal", fill = "teal", alpha = 0.2
    ) +
    coordCartesian(ylim = null to 20000)

4. Ribbon

// Make some more gaps

val dfStep5 = dfStep3
    .update { "uempmed"<Double?>() }
    .where { "date"<LocalDate?>()?.let { d -> d in LocalDate.of(2011, 8, 1)..LocalDate.of(2011, 11, 1) } == true }
    .with { Double.NaN }

val dfStep6 = dfStep5
    .update { "psavert"<Double?>() }
    .where { "date"<LocalDate?>()?.let { d -> d in LocalDate.of(2013, 5, 1)..LocalDate.of(2013, 8, 1) } == true }
    .with { Double.NaN }

val economicsWithGaps2 = dfStep6.toMap()

letsPlot { x = "date"; ymin = "psavert"; ymax = "uempmed" } +
    geomLine(
        data = economics,
        size = 10.0, alpha = 0.1,
        tooltips = tooltipsNone
    ) { y = "psavert" } +
    geomLine(
        data = economics,
        size = 10.0, alpha = 0.1,
        tooltips = tooltipsNone
    ) { y = "uempmed"} +
    geomRibbon(
        data = economicsWithGaps2,
        color = "teal", fill = "teal", alpha = 0.2
    ) +
    geomLabel(
        label = "Missing dates",
        x = epochMillis(2008, 8, 15), y = 13.0,
        nudgeX = -70.0, nudgeUnit = "px"
    ) +
    geomLabel(
        label = "Missing unemployment figures",
        x = epochMillis(2011, 8, 15), y = 24.0,
        nudgeX = 80.0, nudgeUnit = "px"
    ) +
    geomLabel(
        label = "Missing savings rate figures",
        x = epochMillis(2013, 5, 15), y = 3.5,
        nudgeX = 50.0, nudgeUnit = "px"
    ) +
    themeClassic() +
    ggsize(800, 300)