XPages XML Document DataSource - Take 2

For a recent project I revisited the idea of storing XML documents as MIME entries in Notes - while preserving some of the fields for use in views and the Notes client. Jesse suggested I should have a look at annotations. Turns out, it is easier that it sound. To create an annotation that works at runtime, I need a one liner only:
@Retention(RetentionPolicy.RUNTIME) public @interface ItemPathMappings { String[] value(); }
To further improve usefulness, I created a "BaseConfiguration" my classes will inherit from, that contains the common properties I want all my classes (and documents) to have. You might want to adjust it to your needs:

ackage com.notessensei.domino;
import java.io.Serializable;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
/**
 * Common methods implemented by all classes to be Dominoserialized
 */
@XmlRootElement(name = "BaseConfiguration")
@XmlAccessorType(XmlAccessType.NONE)
public abstract class BaseConfiguration implements Serializable, Comparable<BaseConfiguration> {
    private static final long serialVersionUID = 1L;
    @XmlAttribute(name = "name")
    protected String          name;

    public int compareTo(BaseConfiguration bc) {
        return this.toString().compareTo(bc.toString());
    }
    public String getName() {
        return this.name;
    }
    public BaseConfiguration setName(String name) {
        this.name = name;
        return this;
    }
    @Override
    public String toString() {
        return Serializer.toJSONString(this);
    }
    public String toXml() {
        return Serializer.toXMLString(this);
    }
}

The next building block is my Serializer support with a couple of static methods, that make dealing with XML and JSON easier.

package com.notessensei.domino;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;

/**
 * Helper class to serialize / deserialize from/to JSON and XML
 */
public class Serializer {

    public static String toJSONString(Object o) {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        try {
            Serializer.saveJSON(o, out);
        } catch (IOException e) {
            return e.getMessage();
        }
        return out.toString();
    }

    public static String toXMLString(Object o) {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        try {
            Serializer.saveXML(o, out);
        } catch (Exception e) {
            return e.getMessage();
        }
        return out.toString();
    }

    public static void saveJSON(Object o, OutputStream out) throws IOException {
        GsonBuilder gb = new GsonBuilder();
        gb.setPrettyPrinting();
        gb.disableHtmlEscaping();
        Gson gson = gb.create();
        PrintWriter writer = new PrintWriter(out);
        gson.toJson(o, writer);
        writer.flush();
        writer.close();
    }

    public static void saveXML(Object o, OutputStream out) throws Exception {
        JAXBContext context = JAXBContext.newInstance(o.getClass());
        Marshaller m = context.createMarshaller();
        m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
        m.marshal(o, out);
    }

    public static org.w3c.dom.Document getDocument(Object source) throws ParserConfigurationException, JAXBException {
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        DocumentBuilder db = dbf.newDocumentBuilder();
        org.w3c.dom.Document doc = db.newDocument();
        JAXBContext context = JAXBContext.newInstance(source.getClass());
        Marshaller m = context.createMarshaller();
        m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
        m.marshal(source, doc);
        return doc;
    }

    @SuppressWarnings("rawtypes")
    public static Object fromByte(byte[] source, Class targetClass) throws JAXBException {
        ByteArrayInputStream in = new ByteArrayInputStream(source);
        JAXBContext context = JAXBContext.newInstance(targetClass);
        Unmarshaller um = context.createUnmarshaller();
        return targetClass.cast(um.unmarshal(in));
    }
}

The key piece is for the XML serialization/deserialization to work is the abstract class AbstractXmlDocument. That class contains the load and save methods that interact with Domino's MIME capabilities as well as executing the XPath expressions to store the Notes fields. The implementations of this abstract class will have annotations that combine the Notes field name, the type and the XPath expression. An implementation would look like this:

package com.notessensei.domino.xmldocument;
import javax.xml.bind.JAXBException;
import lotus.domino.Database;
import lotus.domino.Document;
import lotus.domino.NotesException;
import lotus.domino.Session;
import com.notessensei.domino.ApplicationConfiguration;
import com.notessensei.domino.Serializer;
import com.notessensei.domino.xmldocument.AbstractXmlDocument.ItemPathMappings;

// The ItemPathMappings are application specific!
@ItemPathMappings({ "Subject|Text|/Application/@name",
     "Description|Text|/Application/description",
     "Unid|Text|/Application/@unid",
     "Audience|Text|/Application/Audiences/Audience",
     "NumberOfViews|Number|count(/Application/Views/View)",
     "NumberOfForms|Number|count(/Application/Forms/Form)",
     "NumberOfColumns|Number|count(/Application/Views/View/columns/column)",
     "NumberOfFields|Number|count(/Application/Forms/Form/fields/field)",
     "NumberOfActions|Number|count(//action)" })
public class ApplicationXmlDocument extends AbstractXmlDocument {

    public ApplicationXmlDocument(String formName) {
        super(formName);
    }

    @SuppressWarnings("unchecked")
    @Override
    public ApplicationConfiguration load(Session session, Document d) {

        ApplicationConfiguration result = null;
        try {
            result = (ApplicationConfiguration) Serializer.fromByte(this.loadFromMime(session, d), ApplicationConfiguration.class);
        } catch (JAXBException e) {
            e.printStackTrace();
        }
        try {
            result.setUnid(d.getUniversalID());
        } catch (NotesException e) {
            // No Action Taken
        }
        return result;
    }

    @SuppressWarnings("unchecked")
    @Override
    public ApplicationConfiguration load(Session session, Database db, String unid) {
        Document doc;
        try {
            doc = db.getDocumentByUNID(unid);
            if (doc != null) {
                ApplicationConfiguration result = this.load(session, doc);
                doc.recycle();
                return result;
            }

        } catch (NotesException e) {
            e.printStackTrace();
        }

        return null;
    }
}

##
Here is the base class:

package com.notessensei.domino.xmldocument;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Scanner;
import java.util.TreeMap;

import javax.xml.bind.JAXBException;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import lotus.domino.Item;
import lotus.domino.MIMEEntity;
import lotus.domino.MIMEHeader;
import lotus.domino.NotesException;
import lotus.domino.Stream;

import org.w3c.dom.Document;
import org.xml.sax.InputSource;

import com.notessensei.domino.BaseConfiguration;
import com.notessensei.domino.Serializer;

@XmlAccessorType(XmlAccessType.NONE)
public abstract class AbstractXmlDocument {

    /**
     * Holds the the definition that extracts data from an XML Document into
     * a Notes Item. Will be triggered by an annotation in the implementation class
     */
    public class FieldExtractDefinition implements Comparable<FieldExtractDefinition> {

        public final static String ELEMENT_SEPARATOR = "\\|";
        private final String       path;
        private final String       itemName;
        private final FieldType    fieldType;

        /**
         * Creates a FieldExtractDefinition from a raw String in the format:
         * Itemname|FieldType|XPath
         *
         * @param rawString
         */
        public FieldExtractDefinition(String rawString) {
            Scanner scanner = new Scanner(rawString);
            scanner.useDelimiter(ELEMENT_SEPARATOR);
            this.itemName = scanner.next();
            this.fieldType = FieldType.valueOf(scanner.next().toUpperCase());
            // The rest of the elements make the XPath
            this.path = scanner.nextLine().substring(1);
        }

        public int compareTo(FieldExtractDefinition fed) {
            return this.toString().compareTo(fed.toString());
        }

        public FieldType getDataType() {
            return fieldType;
        }

        public String getItemName() {
            return itemName;
        }

        public String getXPath() {
            return path;
        }

        @Override
        public String toString() {
            return Serializer.toJSONString(this);
        }

    }

    public enum FieldType {
        NUMBER, TEXT, NAMES, READER, AUTHOR, DATE
    }

    @Retention(RetentionPolicy.RUNTIME)
    public @interface ItemPathMappings {

        String[] value();

    }

    private final String                             MIME_FIELD_NAME = "Body";
    private final String                             FORM_NAME       = "Form";

    private final Collection<FieldExtractDefinition> definitions     = new HashSet<FieldExtractDefinition>();

    private String                                   formName;

    private org.w3c.dom.Document                     style           = null;

    public AbstractXmlDocument(String formName) {
        this.formName = formName;
        // Now load the annotations
        ItemPathMappings ipm = this.getClass().getAnnotation(ItemPathMappings.class);
        if (ipm != null) {
            for (String oneMapping : ipm.value()) {
                this.definitions.add(new FieldExtractDefinition(oneMapping));
            }
        }
    }

    public String getFormName() {
        return this.formName;
    }

    public org.w3c.dom.Document getStyle() {
        return this.style;
    }

    public abstract <T extends BaseConfiguration> T load(lotus.domino.Session session, lotus.domino.Database db, String unid);

    public abstract <T extends BaseConfiguration> T load(lotus.domino.Session session, lotus.domino.Document d);

    public boolean save(lotus.domino.Session session, lotus.domino.Document d, Object source) {
        boolean success = true;
        try {
            boolean oldMime = session.isConvertMime();
            session.setConvertMime(false);

            // Save the form to be sure
            if (this.formName != null) {
                d.replaceItemValue(FORM_NAME, this.formName);
            }

            // We must remove the mime body - otherwise it will not work
            if (d.hasItem(MIME_FIELD_NAME)) {
                d.removeItem(MIME_FIELD_NAME);
            }

            org.w3c.dom.Document domDoc = Serializer.getDocument(source);

            // Create the top mime entry
            MIMEEntity root = d.createMIMEEntity(MIME_FIELD_NAME);
            MIMEHeader header = root.createHeader("Content-Type");
            header.setHeaderVal("multipart/mixed");

            MIMEEntity body = root.createChildEntity();
            MIMEHeader bheader = body.createHeader("Content-Type");
            bheader.setHeaderVal("multipart/alternative");

            MIMEEntity textMime = body.createChildEntity();
            MIMEHeader textHeader = textMime.createHeader("Content-Type");
            String cType = (this.style == null) ? "text/plain" : "text/html";
            textHeader.setHeaderVal(cType);

            Stream stream2 = session.createStream();
            stream2.write(this.dom2Byte(domDoc, this.style));
            textMime.setContentFromBytes(stream2, cType + "; charset=\"UTF-8\"", MIMEEntity.ENC_NONE);

            MIMEEntity xmlMime = body.createChildEntity();
            MIMEHeader xmlHeader = xmlMime.createHeader("Content-Type");
            xmlHeader.setHeaderVal("application/xml");

            Stream stream = session.createStream();
            stream.write(this.dom2Byte(domDoc, null));
            xmlMime.setContentFromBytes(stream, "application/xml; charset=\"UTF-8\"", MIMEEntity.ENC_NONE);

            // Now extract the fields
            this.extractFields(domDoc, d, body);

            session.setConvertMime(oldMime);
        } catch (NotesException e) {
            success = false;
            e.printStackTrace();
        } catch (ParserConfigurationException e) {
            success = false;
            e.printStackTrace();
        } catch (JAXBException e) {
            success = false;
            e.printStackTrace();
        }

        return success;
    }

    public Map<String, String> extractMappingResults(Object source) throws ParserConfigurationException, JAXBException {
        Map<String, String> result = new TreeMap<String, String>();
        org.w3c.dom.Document domDoc = (source instanceof org.w3c.dom.Document) ? (org.w3c.dom.Document) source : Serializer
                    .getDocument(source);
        for (FieldExtractDefinition f : this.definitions) {
            String candidate = this.extractOneField(domDoc, f.getXPath());
            if (candidate != null) {
                result.put(f.getItemName(), candidate);
            }
        }

        return result;
    }

    public void setFormName(String formName) {
        this.formName = formName;
    }

    public AbstractXmlDocument setPrettyPrintStyle(org.w3c.dom.Document style) {
        this.style = style;
        return this;
    }

    public void setStyle(InputStream styleStream) {

        Document d = null;
        final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        factory.setValidating(false); // Will blow if set to true
        factory.setNamespaceAware(true);

        try {
            InputSource source = new InputSource(styleStream);
            final DocumentBuilder docb = factory.newDocumentBuilder();
            d = docb.parse(source);

        } catch (final Exception e) {
            e.printStackTrace();
            d = null;
        }

        if (d == null) {
            System.out.println("DOM from stream generation failed:\n");
        }

        this.style = d;
    }

    /**
     * Loads XML from Notes document and returns the content as bytes to
     * be marshalled into a Java object creation
     *
     * @param session
     *            NotesSession
     * @param d
     *            Document with MIME content
     * @return bytes for object instance creation
     */
    protected byte[] loadFromMime(lotus.domino.Session session, lotus.domino.Document d) {
        ByteArrayOutputStream out = new ByteArrayOutputStream();

        try {
            boolean oldMime = session.isConvertMime();
            session.setConvertMime(false);

            // Load the TOP mime entitiy - must be multipart/mixed
            MIMEEntity root = d.getMIMEEntity(MIME_FIELD_NAME);
            if (root != null && root.getContentType().equals("multipart") && root.getContentSubType().equals("mixed")) {
                // We check for the body whtou should be multipart/alternate
                MIMEEntity body = root.getFirstChildEntity();
                if (body != null && body.getContentType().equals("multipart") && body.getContentSubType().equals("alternative")) {
                    // Now we go after Content-Type = application/xml
                    MIMEEntity payload = body.getFirstChildEntity();

                    while (payload != null && !payload.getContentSubType().equals("xml")) {
                        System.out.println(payload.getContentType());
                        System.out.println(payload.getContentSubType());
                        MIMEEntity nextMime = payload.getNextEntity();
                        payload.recycle();
                        payload = nextMime;
                    }

                    if (payload != null) {
                        Stream notesStream = session.createStream();
                        payload.getContentAsBytes(notesStream);
                        notesStream.setPosition(0);
                        notesStream.getContents(out);
                        notesStream.recycle();
                    }
                }

            }
            ;
            session.setConvertMime(oldMime);
        } catch (NotesException e) {
            e.printStackTrace();
        }

        return out.toByteArray();
    }

    private byte[] dom2Byte(Document dom, Document stylesheet) {
        StreamResult xResult = null;
        DOMSource source = null;
        final ByteArrayOutputStream out = new ByteArrayOutputStream();
        Transformer transformer = null;

        try {
            final TransformerFactory tFactory = TransformerFactory.newInstance();
            xResult = new StreamResult(out);
            source = new DOMSource(dom);

            if (stylesheet != null) {
                final DOMSource style = new DOMSource(stylesheet);
                transformer = tFactory.newTransformer(style);
            } else {
                transformer = tFactory.newTransformer();
            }

            // We don't want the XML declaration in front
            transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");

            transformer.transform(source, xResult);

        } catch (final Exception e) {
            e.printStackTrace();
        }

        return out.toByteArray();

    }

    private void extractFields(Document domDoc, lotus.domino.Document d, MIMEEntity body) {
        Map<String, String> itemList = null;
        try {
            itemList = this.extractMappingResults(domDoc);
        } catch (ParserConfigurationException e1) {
            e1.printStackTrace();
        } catch (JAXBException e1) {
            e1.printStackTrace();
        }

        if (itemList == null) {
            return;
        }

        for (FieldExtractDefinition f : this.definitions) {

            String fName = f.getItemName();
            String fValue = itemList.get(fName);

            if (!fName.equalsIgnoreCase(MIME_FIELD_NAME) && !(fValue == null)) {
                // We can't allow to overwrite the body or write empty values
                try {
                    MIMEHeader h = body.createHeader(fName);
                    h.setHeaderVal(fValue);
                    Item curItem = d.replaceItemValue(fName, fValue);
                    FieldType ft = f.getDataType();
                    if (ft.equals(FieldType.AUTHOR)) {
                        curItem.setAuthors(true);
                    } else if (ft.equals(FieldType.READER)) {
                        curItem.setReaders(true);
                    }

                } catch (NotesException e) {
                    e.printStackTrace();
                }
            }

        }
    }

    private String extractOneField(Document domDoc, String xPathString) {

        Object exprResult = null;
        final XPath xpath = XPathFactory.newInstance().newXPath();
        final MagicNamespaceContext nsc = new MagicNamespaceContext();
        xpath.setNamespaceContext(nsc);

        try {
            exprResult = xpath.evaluate(xPathString, domDoc);
        } catch (final XPathExpressionException e) {
            System.err.println("XPATH failed for " + xPathString);
            System.err.println(e.getMessage());
        }
        // if it failed we abort
        if (exprResult == null) {
            return null;
        }

        return exprResult.toString();

    }
}

As time permits I will provide a full working example.
YMMV!