\label{doc_decisions_1_in_discussion_internal_cache_md_md_doc_decisions_1_in_discussion_internal_cache}%
\Hypertarget{doc_decisions_1_in_discussion_internal_cache_md_md_doc_decisions_1_in_discussion_internal_cache}%
 \hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1862}{}\doxysection{Problem}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1862}
{\ttfamily kdb\+Get} might return more or fewer keys than requested. This was found confusing several times.\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1863}{}\doxysubsection{More Keys}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1863}
Even if calling {\ttfamily kdb\+Get} with a parent key below a mountpoint, {\ttfamily kdb\+Get} will nevertheless return all keys of the mountpoint. Pseudo code example, assuming there is a mountpoint at {\ttfamily /mountpoint} and a key {\ttfamily /mountpoint/other}\+:


\begin{DoxyCode}{0}
\DoxyCodeLine{kdbGet (kdb, ks, keyNew("{}/mountpoint/below"{}));}
\DoxyCodeLine{assert (ksLookup (ks, "{}/mountpoint/other"{}) == NULL);}

\end{DoxyCode}


It was found unexpected that this assert will fail.\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1864}{}\doxysubsection{Fewer Keys}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1864}
When doing a second {\ttfamily kdb\+Get} with a new keyset no keys will be returned when no backends report changed data, because kdb internally thinks the data is already up-\/to-\/date\+:


\begin{DoxyCode}{0}
\DoxyCodeLine{\textcolor{keyword}{static} \textcolor{keywordtype}{void} test\_doubleGet (\textcolor{keywordtype}{void})}
\DoxyCodeLine{\{}
\DoxyCodeLine{        printf(\textcolor{stringliteral}{"{}running \%s\(\backslash\)n"{}}, \_\_func\_\_);}
\DoxyCodeLine{}
\DoxyCodeLine{        \textcolor{comment}{// Setup}}
\DoxyCodeLine{        Key * parentKey = \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}}(\textcolor{stringliteral}{"{}/somewhere"{}}, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{        KeySet * ks = \mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}}(0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{        KDB * \mbox{\hyperlink{namespacekdb}{kdb}} = \mbox{\hyperlink{group__kdb_ga844e1299a84c3fbf1d3a905c5c893ba5}{kdbOpen}} (\mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}}(0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}}), parentKey);}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga28e385fd9cb7ccfe0b2f1ed2f62453a1}{kdbGet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks, parentKey);}
\DoxyCodeLine{        \mbox{\hyperlink{group__keyset_gaa5a1d467a4d71041edce68ea7748ce45}{ksAppendKey}} (ks, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}} (\textcolor{stringliteral}{"{}user:/somewhere"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8ac66e4a49d09212b79f5754ca6db5bd2e}{KEY\_VALUE}}, \textcolor{stringliteral}{"{}abc"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        \mbox{\hyperlink{group__keyset_gaa5a1d467a4d71041edce68ea7748ce45}{ksAppendKey}} (ks, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}} (\textcolor{stringliteral}{"{}user:/somewhere/key"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8ac66e4a49d09212b79f5754ca6db5bd2e}{KEY\_VALUE}}, \textcolor{stringliteral}{"{}xyz"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga11436b058408f83d303ca5e996832bcf}{kdbSet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks, parentKey);}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_gadb54dc9fda17ee07deb9444df745c96f}{kdbClose}} (\mbox{\hyperlink{namespacekdb}{kdb}}, parentKey);}
\DoxyCodeLine{}
\DoxyCodeLine{        \textcolor{comment}{// Scenario}}
\DoxyCodeLine{        \mbox{\hyperlink{namespacekdb}{kdb}} = \mbox{\hyperlink{group__kdb_ga844e1299a84c3fbf1d3a905c5c893ba5}{kdbOpen}} (\mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}}(0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}}), parentKey);}
\DoxyCodeLine{}
\DoxyCodeLine{        KeySet * ks1 = \mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}} (0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{        KeySet * ks2 = \mbox{\hyperlink{group__keyset_ga671e1aaee3ae9dc13b4834a4ddbd2c3c}{ksNew}} (0, \mbox{\hyperlink{group__keyset_ga7a28fce3773b2c873c94ac80b8b4cd54}{KS\_END}});}
\DoxyCodeLine{}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga28e385fd9cb7ccfe0b2f1ed2f62453a1}{kdbGet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks1, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}}(\textcolor{stringliteral}{"{}/somewhere"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        succeed\_if (\mbox{\hyperlink{group__keyset_gad65d2cdcbb5381194a1688e169af8a83}{ksLookupByName}} (ks1, \textcolor{stringliteral}{"{}/somewhere/key"{}}, 0) != NULL, \textcolor{stringliteral}{"{}should find key (1)"{}});}
\DoxyCodeLine{        \mbox{\hyperlink{group__kdb_ga28e385fd9cb7ccfe0b2f1ed2f62453a1}{kdbGet}} (\mbox{\hyperlink{namespacekdb}{kdb}}, ks2, \mbox{\hyperlink{group__key_gad23c65b44bf48d773759e1f9a4d43b89}{keyNew}}(\textcolor{stringliteral}{"{}/somewhere"{}}, \mbox{\hyperlink{group__key_gga9b703ca49f48b482def322b77d3e6bc8aa8adb6fcb92dec58fb19410eacfdd403}{KEY\_END}}));}
\DoxyCodeLine{        succeed\_if (\mbox{\hyperlink{group__keyset_gad65d2cdcbb5381194a1688e169af8a83}{ksLookupByName}} (ks2, \textcolor{stringliteral}{"{}/somewhere/key"{}}, 0) != NULL, \textcolor{stringliteral}{"{}should find key (2)"{}});}
\DoxyCodeLine{\}}

\end{DoxyCode}


It was found unexpected that the second assert -\/ should find key (2) -\/ will fail.\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1865}{}\doxysection{Constraints}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1865}

\begin{DoxyItemize}
\item memory consumption must be low for {\ttfamily kdb\+Get}, see \mbox{\hyperlink{doc_GOALS_md}{4. Goal\+: Performance}}, in particular, deep duplication is too expensive
\item For non-\/optional parts of Elektra, also non-\/\+POSIX systems must be supported
\end{DoxyItemize}\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1866}{}\doxysection{Assumptions}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1866}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1867}{}\doxysection{Considered Alternatives}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1867}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1868}{}\doxysubsection{Keep Current Situation}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1868}
Improve documentation to make people more aware of these two problems\+:


\begin{DoxyItemize}
\item add a tutorial about {\ttfamily kdb\+Get} semantics
\item add full examples how to correctly work with {\ttfamily kdb\+Get}
\end{DoxyItemize}\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1869}{}\doxysubsection{Simple Solution by @kodebach}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1869}
\char`\"{}\+More keys\char`\"{}\+:

Upon returning from {\ttfamily kdb\+Get}/{\ttfamily kdb\+Set}, set the keyname of parent\+Key to the key that actually is parent of the data that was loaded. I.\+e. to the mountpoint of the backend that contains parent\+Key.

\begin{quote}
@markus2330 and @atmaxinger find this behavior very unexpected \end{quote}
\char`\"{}\+Fewer keys\char`\"{}\+:

IMO this is simply a bug in the \char`\"{}nothing changed\char`\"{} logic. I don\textquotesingle{}t think we need some complicated COW scheme to solve this. I haven\textquotesingle{}t had a chance to check this yet, but I think it is just a matter of copying the keys from {\ttfamily backend\+Data-\/$>$keys}, so they are actually there and we don\textquotesingle{}t just assume they are there.\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1870}{}\doxysubsection{Cachefilter Plugin}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1870}
Naively one would simply cache the whole keyset and use {\ttfamily ks\+Below} to always get the keyset.

This idea was implemented and later on \href{https://github.com/ElektraInitiative/libelektra/commit/a3d95f07160d792fdd0f169d8543138c32a2f580}{\texttt{ discarded\+: a3d95f}}

The main problems are\+:


\begin{DoxyItemize}
\item very high memory consumption (duplication of Key\+Sets)
\item problems specific to \mbox{\hyperlink{doc_decisions_4_partially_implemented_hooks_md}{hooks}}, see \href{https://issues.libelektra.org/1072}{\texttt{ \#1072}}
\end{DoxyItemize}\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1871}{}\doxysubsection{MMAP Cache with parent key}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1871}
We make the mmap cache non-\/optional so that we always have a keyset of configuration data internally. The cache will be used to do change tracking. From this keyset, we use {\ttfamily ks\+Below} to return the correct keyset.


\begin{DoxyItemize}
\item Disadvantage\+: mmap implementation for Windows would be needed
\end{DoxyItemize}

@kodebach wrote\+:

\begin{quote}
I\textquotesingle{}m not entirely sure this is possible (@mpranj may know more), but the way I understand it the cache should be updated at the end of every kdb\+Get that was a cache miss. So during kdb\+Set (assuming there is no external modification, i.\+e. conflict) the on-\/disk data of the cache should always be up-\/to-\/date. My idea would be to just read the cached keyset from disk and diff against the current keyset in kdb\+Set.

This would mean enabling change tracking also enables the cache (or at least updating the cache, we don\textquotesingle{}t have to use it in kdb\+Get). If that\textquotesingle{}s not wanted or if the cache data cannot be used directly for some other reason, the same approach could still be used. We\textquotesingle{}d just have to write the keyset to disk during kdb\+Get and use it during kdb\+Set. Since disk space is far less precious than RAM, we could even create separate files for every parent key. If we do go down this route, kdb\+Close should cleanup the files created by this KDB instance to avoid wasting disk space.

This approach wouldn\textquotesingle{}t be very performant (since it uses disk IO), but especially if we can use the cache data, it should be pretty easy to do. \end{quote}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1872}{}\doxysubsection{MMAP Cache without parent key}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1872}
We make the mmap cache non-\/optional and only use a single cache, caching everything. We remove the parent key of {\ttfamily kdb\+Get} and {\ttfamily kdb\+Set} and always return the keyset of the whole KDB.


\begin{DoxyItemize}
\item Disadvantage\+: mmap implementation for Windows would be needed
\item Then also symlinks (fallback, override etc.) and constraints for keys outside of the parent\+Key would work. It would make the {\ttfamily -\/a} option of {\ttfamily kdb get} unnecessary.
\end{DoxyItemize}

@kodebach wrote (in response to the \textquotesingle{}symlinks\textquotesingle{} point)\+:

\begin{quote}
You still haven\textquotesingle{}t addressed why this would actually be a good default behavior. You just claim it is an advantage without any explanation.

Like I said, I would normally expect mountpoints to be isolated. If symlinks work like this, the isolation is partially broken. One could argue that the problem right now is that such \char`\"{}broken\char`\"{} symlinks are not prevented. But your solution doesn\textquotesingle{}t completely fix the problem either.

Take a variant of my previous example\+:


\begin{DoxyItemize}
\item {\ttfamily system\+:/foo} is a mountpoint, there are no other mountpoints
\item {\ttfamily spec\+:/foo/bar} refers to {\ttfamily system\+:/abc} via {\ttfamily meta\+:/override}
\end{DoxyItemize}

Now in a plugin that is part of the mountpoint {\ttfamily system\+:/foo} doing a {\ttfamily ks\+Lookup\+By\+Name (ks, \char`\"{}/foo/bar\char`\"{}, 0)} would NOT use {\ttfamily system\+:/abc}, because that key is never part of the keyset passed to this plugin.

However, in an application that used {\ttfamily system\+:/foo} as the parent, the {\ttfamily meta\+:/override} would work with your proposal. To me that seems like very confusing behavior, because both the application and the plugin seemingly use the same parent key. \end{quote}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1873}{}\doxysubsection{Data restrictions}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1873}
@kodebach wrote\+:

\begin{quote}
Make all the keys returned by kdb\+Get completely read-\/only. To change the data you need to append an entirely new key to replace the existing one. Then we just need to keep a shallow copy internally. \end{quote}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1874}{}\doxysubsection{API restrictions}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1874}
@kodebach wrote\+:

\begin{quote}
Change the API and remove Key\+Set from kdb\+Get and kdb\+Set also option 4 in \mbox{\hyperlink{doc_decisions_0_drafts_operation_sequences_md}{the operation sequences decision}}. If the keyset is owned by the KDB handle, it should not be as big surprise, if there is extra data in there. I certainly wouldn\textquotesingle{}t try to assert anything on the contents of a Key\+Set that I don\textquotesingle{}t own directly, unless the condition is explicitly documented somewhere. \end{quote}
@markus2330 wrote\+:

\begin{quote}
I disagree, it is actually the same kind of surprise for \char`\"{}\+More Keys\char`\"{}. Only the \char`\"{}\+Fewer Keys\char`\"{} would get fixed. \end{quote}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1875}{}\doxysubsection{Copy On Write}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1875}
We keep a duplicated keyset in-\/memory and tag the keys as copy-\/on-\/write (COW). From this keyset, we use {\ttfamily ks\+Below} to return the correct keyset. If the user tries to change the value or metadata of these keys, the data gets duplicated. I.\+e. the original keyset is not changed. The name is not relevant. It is always read-\/only, because the key is in at least one keyset (the internal one).

Possible copy-\/on-\/write implementations are described in \mbox{\hyperlink{doc_decisions_1_in_discussion_copy_on_write_md}{another decision}}.\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1876}{}\doxysection{Decision}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1876}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1877}{}\doxysection{Rationale}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1877}
Semantics can be provided without additional code or overhead in the core.\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1878}{}\doxysection{Implications}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1878}
\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1879}{}\doxysection{Related Decisions}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1879}

\begin{DoxyItemize}
\item \mbox{\hyperlink{doc_decisions_0a_delayed_global_validation_md}{Global Validation}}
\item \mbox{\hyperlink{doc_decisions_1_in_discussion_copy_on_write_md}{Copy On Write}}
\end{DoxyItemize}\hypertarget{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1880}{}\doxysection{Notes}\label{doc_decisions_1_in_discussion_internal_cache_md_autotoc_md1880}
Problem \char`\"{}\+Fewer Keys\char`\"{} was found to be a \href{https://pull.libelektra.org/4619}{\texttt{ \char`\"{}horrible problem\char`\"{}}}

Issues where the problem described here was found confusing\+:


\begin{DoxyItemize}
\item \href{https://issues.libelektra.org/760}{\texttt{ \#760}}
\item \href{https://issues.libelektra.org/1363}{\texttt{ \#1363}}
\end{DoxyItemize}

@mpranj wrote about the performance for {\ttfamily MMAP Cache without parent key}\+:

\begin{quote}
in my benchmarks the pointer correction was never a bottleneck. \end{quote}
