\label{doc_decisions_internal_cache_md_md_doc_decisions_internal_cache}%
\Hypertarget{doc_decisions_internal_cache_md_md_doc_decisions_internal_cache}%
 \hypertarget{doc_decisions_internal_cache_md_autotoc_md2026}{}\doxysection{Problem}\label{doc_decisions_internal_cache_md_autotoc_md2026}
{\ttfamily kdb\+Get} might return more or fewer keys than requested. This was found confusing several times.\hypertarget{doc_decisions_internal_cache_md_autotoc_md2027}{}\doxysubsection{More Keys}\label{doc_decisions_internal_cache_md_autotoc_md2027}
Even if calling {\ttfamily kdb\+Get} with a parent key below a mountpoint, {\ttfamily kdb\+Get} will nevertheless return all keys of the mountpoint. Pseudo code example, assuming there is a mountpoint at {\ttfamily /mountpoint} and a key {\ttfamily /mountpoint/other}\+:


\begin{DoxyCode}{0}
\DoxyCodeLine{kdbGet (kdb, ks, keyNew("{}/mountpoint/below"{}));}
\DoxyCodeLine{assert (ksLookup (ks, "{}/mountpoint/other"{}) == NULL);}

\end{DoxyCode}


It was found unexpected that this assert will fail.\hypertarget{doc_decisions_internal_cache_md_autotoc_md2028}{}\doxysubsection{Fewer Keys}\label{doc_decisions_internal_cache_md_autotoc_md2028}
When doing a second {\ttfamily kdb\+Get} with a new keyset no keys will be returned when no backends report changed data, because kdb internally thinks the data is already up-\/to-\/date. A unit test by @atmaxinger\+:


\begin{DoxyCode}{0}
\DoxyCodeLine{\textcolor{keyword}{static} \textcolor{keywordtype}{void} test\_doubleGet (\textcolor{keywordtype}{void})}
\DoxyCodeLine{\{}
\DoxyCodeLine{        printf(\textcolor{stringliteral}{"{}running \%s\(\backslash\)n"{}}, \_\_func\_\_);}
\DoxyCodeLine{}
\DoxyCodeLine{        \textcolor{comment}{// Setup}}
\DoxyCodeLine{        Key * parentKey = \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}}(\textcolor{stringliteral}{"{}/somewhere"{}}, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{        KeySet * ks = \mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}}(0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{        KDB * \mbox{\hyperlink{namespacekdb}{kdb}} = \mbox{\hyperlink{group__kdb_ga844e1299a84c3fbf1d3a905c5c893ba5}{kdbOpen}} (\mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}}(0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}}), parentKey);}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga28e385fd9cb7ccfe0b2f1ed2f62453a1}{kdbGet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks, parentKey);}
\DoxyCodeLine{        \mbox{\hyperlink{group__keyset_gaa5a1d467a4d71041edce68ea7748ce45}{ksAppendKey}} (ks, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}} (\textcolor{stringliteral}{"{}user:/somewhere"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8ac66e4a49d09212b79f5754ca6db5bd2e}{KEY\_VALUE}}, \textcolor{stringliteral}{"{}abc"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        \mbox{\hyperlink{group__keyset_gaa5a1d467a4d71041edce68ea7748ce45}{ksAppendKey}} (ks, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}} (\textcolor{stringliteral}{"{}user:/somewhere/key"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8ac66e4a49d09212b79f5754ca6db5bd2e}{KEY\_VALUE}}, \textcolor{stringliteral}{"{}xyz"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga11436b058408f83d303ca5e996832bcf}{kdbSet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks, parentKey);}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_gadb54dc9fda17ee07deb9444df745c96f}{kdbClose}} (\mbox{\hyperlink{namespacekdb}{kdb}}, parentKey);}
\DoxyCodeLine{}
\DoxyCodeLine{        \textcolor{comment}{// Scenario}}
\DoxyCodeLine{        \mbox{\hyperlink{namespacekdb}{kdb}} = \mbox{\hyperlink{group__kdb_ga844e1299a84c3fbf1d3a905c5c893ba5}{kdbOpen}} (\mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}}(0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}}), parentKey);}
\DoxyCodeLine{}
\DoxyCodeLine{        KeySet * ks1 = \mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}} (0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{        KeySet * ks2 = \mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}} (0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga28e385fd9cb7ccfe0b2f1ed2f62453a1}{kdbGet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks1, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}}(\textcolor{stringliteral}{"{}/somewhere"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        succeed\_if (\mbox{\hyperlink{group__keyset_gad65d2cdcbb5381194a1688e169af8a83}{ksLookupByName}} (ks1, \textcolor{stringliteral}{"{}/somewhere/key"{}}, 0) != NULL, \textcolor{stringliteral}{"{}should find key (1)"{}});}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga28e385fd9cb7ccfe0b2f1ed2f62453a1}{kdbGet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks2, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}}(\textcolor{stringliteral}{"{}/somewhere"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        succeed\_if (\mbox{\hyperlink{group__keyset_gad65d2cdcbb5381194a1688e169af8a83}{ksLookupByName}} (ks2, \textcolor{stringliteral}{"{}/somewhere/key"{}}, 0) != NULL, \textcolor{stringliteral}{"{}should find key (2)"{}});}
\DoxyCodeLine{\}}

\end{DoxyCode}


\begin{quote}
It actually outputs should find key (2) so the assertion fails. \end{quote}
It was found unexpected that the second assert will fail.\hypertarget{doc_decisions_internal_cache_md_autotoc_md2029}{}\doxysection{Constraints}\label{doc_decisions_internal_cache_md_autotoc_md2029}

\begin{DoxyItemize}
\item memory consumption must be low for {\ttfamily kdb\+Get}, see \mbox{\hyperlink{doc_GOALS_md}{4. Goal\+: Performance}}, in particular, deep duplication is too expensive
\end{DoxyItemize}\hypertarget{doc_decisions_internal_cache_md_autotoc_md2030}{}\doxysection{Assumptions}\label{doc_decisions_internal_cache_md_autotoc_md2030}
\hypertarget{doc_decisions_internal_cache_md_autotoc_md2031}{}\doxysection{Considered Alternatives}\label{doc_decisions_internal_cache_md_autotoc_md2031}
\hypertarget{doc_decisions_internal_cache_md_autotoc_md2032}{}\doxysubsection{Keep Current Situation}\label{doc_decisions_internal_cache_md_autotoc_md2032}
Improve documentation to make people more aware of these two problems\+:


\begin{DoxyItemize}
\item add a tutorial about {\ttfamily kdb\+Get} semantics
\item add full examples how to correctly work with {\ttfamily kdb\+Get}
\end{DoxyItemize}\hypertarget{doc_decisions_internal_cache_md_autotoc_md2033}{}\doxysubsection{Cachefilter Plugin}\label{doc_decisions_internal_cache_md_autotoc_md2033}
Naively one would simply cache the whole keyset and use {\ttfamily ks\+Below} to always get the keyset.

This idea was implemented and later on discarded\+: a3d95f07160d792fdd0f169d8543138c32a2f580

The main problems are\+:


\begin{DoxyItemize}
\item very high memory consumption (duplication of Key\+Sets)
\item problems specific to \mbox{\hyperlink{doc_decisions_hooks_md}{hooks}}, see \href{https://issues.libelektra.org/1072}{\texttt{ \#1072}}
\end{DoxyItemize}\hypertarget{doc_decisions_internal_cache_md_autotoc_md2034}{}\doxysubsection{MMAP Cache with parent key}\label{doc_decisions_internal_cache_md_autotoc_md2034}
We make the mmap cache non-\/optional so that we always have a keyset of configuration data internally. From this keyset, we use {\ttfamily ks\+Below} to return the correct keyset.

{\bfseries{Cons\+:}}


\begin{DoxyItemize}
\item invalidation of OPMPHM
\end{DoxyItemize}\hypertarget{doc_decisions_internal_cache_md_autotoc_md2035}{}\doxysubsection{MMAP Cache without parent key}\label{doc_decisions_internal_cache_md_autotoc_md2035}
We make the mmap cache non-\/optional and only use a single cache, caching everything. We remove the parent key of {\ttfamily kdb\+Get} and {\ttfamily kdb\+Set} and always return the keyset of the whole KDB.\hypertarget{doc_decisions_internal_cache_md_autotoc_md2036}{}\doxysubsection{In-\/\+Memory COW Cache}\label{doc_decisions_internal_cache_md_autotoc_md2036}
We keep a duplicated keyset in-\/memory and tag the keys as copy-\/on-\/write (COW). From this keyset, we use {\ttfamily ks\+Below} to return the correct keyset. If the user tries to change the value or metadata of these keys, the data gets duplicated. I.\+e. the original keyset is not changed. The name is not relevant. It is always read-\/only, because the key is in at least one keyset (the internal one). Pseudo code example\+:


\begin{DoxyCode}{0}
\DoxyCodeLine{Key * key = \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}} (\textcolor{stringliteral}{"{}dir:/something"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8ac66e4a49d09212b79f5754ca6db5bd2e}{KEY\_VALUE}}, \textcolor{stringliteral}{"{}my value"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}});}
\DoxyCodeLine{\mbox{\hyperlink{group__key_ga505575ebef060066984fe0f590081e37}{keyCopy}} (key\_dup, key, ELEKTRA\_CP\_COW);}
\DoxyCodeLine{assert (\mbox{\hyperlink{group__keyvalue_ga880936f2481d28e6e2acbe7486a21d05}{keyString}}(key) == \mbox{\hyperlink{group__keyvalue_ga880936f2481d28e6e2acbe7486a21d05}{keyString}}(key\_dup));}
\DoxyCodeLine{\mbox{\hyperlink{group__keyvalue_ga622bde1eb0e0c4994728331326340ef2}{keySetString}} (key\_dup, \textcolor{stringliteral}{"{}other value"{}}); \textcolor{comment}{// COW done here}}
\DoxyCodeLine{assert (\mbox{\hyperlink{group__keyvalue_ga880936f2481d28e6e2acbe7486a21d05}{keyString}}(key) != \mbox{\hyperlink{group__keyvalue_ga880936f2481d28e6e2acbe7486a21d05}{keyString}}(key\_dup));}
\DoxyCodeLine{assert (\mbox{\hyperlink{group__keyname_ga7699091610e7f3f43d2949514a4b35d9}{keySetName}} (key\_dup, \textcolor{stringliteral}{"{}dir:/valid"{}}) == -\/1); \textcolor{comment}{// must fail, as we have a COW key}}
\DoxyCodeLine{assert (\mbox{\hyperlink{group__keyname_ga8e805c726a60da921d3736cda7813513}{keyName}}(key) == \mbox{\hyperlink{group__keyname_ga8e805c726a60da921d3736cda7813513}{keyName}}(key\_dup)); \textcolor{comment}{// stays always valid}}

\end{DoxyCode}


This is already implemented for the MMAP cache, so the implementation should be straightforward\+: Do the same COW duplications as done for MMAP but with a different flag.

@kodebach wrote\+:

\begin{quote}
What I wanted to say is that mmap already does COW, so we can reuse the code and probably the flag. If there is some code that is only needed for mmap and not COW, we could make mmap set two flags one for mmap and one for the general COW code. \end{quote}
For the metadata, however, also COW Key\+Sets might be needed (at least with the current API). Example\+:


\begin{DoxyCode}{0}
\DoxyCodeLine{\mbox{\hyperlink{group__key_ga505575ebef060066984fe0f590081e37}{keyCopy}} (cow, key, ELEKTRA\_CP\_COW);}
\DoxyCodeLine{KeySet * cowMeta = \mbox{\hyperlink{group__keymeta_ga11706f1753e67933f7cffc5c0345cd29}{keyMeta}} (cow);}
\DoxyCodeLine{\mbox{\hyperlink{group__keyset_gaa5a1d467a4d71041edce68ea7748ce45}{ksAppendKey}} (cowMeta, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}} (\textcolor{stringliteral}{"{}meta:/whatever"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8ac66e4a49d09212b79f5754ca6db5bd2e}{KEY\_VALUE}}, \textcolor{stringliteral}{"{}abc"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{ksRemoveByName (cowMeta, \textcolor{stringliteral}{"{}meta:/type"{}});}

\end{DoxyCode}


{\bfseries{Pros\+:}}


\begin{DoxyItemize}
\item Elektra doesn\textquotesingle{}t require MMAP
\end{DoxyItemize}\hypertarget{doc_decisions_internal_cache_md_autotoc_md2037}{}\doxysubsection{Data restrictions}\label{doc_decisions_internal_cache_md_autotoc_md2037}
@kodebach wrote\+:

\begin{quote}
Make all the keys returned by kdb\+Get completely read-\/only. To change the data you need to append an entirely new key to replace the existing one. Then we just need to keep a shallow copy internally. \end{quote}
\hypertarget{doc_decisions_internal_cache_md_autotoc_md2038}{}\doxysubsection{API restrictions}\label{doc_decisions_internal_cache_md_autotoc_md2038}
@kodebach wrote\+:

\begin{quote}
Change the API and remove Key\+Set from kdb\+Get and kdb\+Set also option 4 in \mbox{\hyperlink{doc_decisions_operation_sequences_md}{the operation sequences decision}}. If the keyset is owned by the KDB handle, it should not be as big surprise, if there is extra data in there. I certainly wouldn\textquotesingle{}t try to asset anything on the contents of a Key\+Set that I don\textquotesingle{}t own directly, unless the condition is explicitly documented somewhere. \end{quote}
@markus2330 wrote\+:

\begin{quote}
I disagree, it is actually the same kind of surprise for \char`\"{}\+More Keys\char`\"{}. Only the \char`\"{}\+Fewer Keys\char`\"{} would get fixed. \end{quote}
\hypertarget{doc_decisions_internal_cache_md_autotoc_md2039}{}\doxysection{Decision}\label{doc_decisions_internal_cache_md_autotoc_md2039}
Not yet decided.\hypertarget{doc_decisions_internal_cache_md_autotoc_md2040}{}\doxysection{Rationale}\label{doc_decisions_internal_cache_md_autotoc_md2040}
Semantics can be provided without additional code or overhead in the core.\hypertarget{doc_decisions_internal_cache_md_autotoc_md2041}{}\doxysection{Implications}\label{doc_decisions_internal_cache_md_autotoc_md2041}
\hypertarget{doc_decisions_internal_cache_md_autotoc_md2042}{}\doxysection{Related Decisions}\label{doc_decisions_internal_cache_md_autotoc_md2042}

\begin{DoxyItemize}
\item \mbox{\hyperlink{doc_decisions_global_validation_md}{Global Validation}}
\end{DoxyItemize}\hypertarget{doc_decisions_internal_cache_md_autotoc_md2043}{}\doxysection{Notes}\label{doc_decisions_internal_cache_md_autotoc_md2043}
Problem \char`\"{}\+Fewer Keys\char`\"{} was found to be a \href{https://pull.libelektra.org/4619}{\texttt{ \char`\"{}horrible problem\char`\"{}}}

Issues where the problem described here was found confusing\+:


\begin{DoxyItemize}
\item \href{https://issues.libelektra.org/760}{\texttt{ \#760}}
\item \href{https://issues.libelektra.org/1363}{\texttt{ \#1363}}
\end{DoxyItemize}

@mpranj wrote about the performance for {\ttfamily MMAP Cache without parent key}\+:

\begin{quote}
in my benchmarks the pointer correction was never a bottleneck. \end{quote}
