Selection exclusion in nested QGraphicsScenes

Disclamer

I personally do not recommend embedding widgets in a QGraphicsScene:

The key to the high performance of Graphics View is reducing how much is painted each frame. QGraphicsWidget and QGraphicsProxyWidget together are huge performance killers because they can not be rendered in a efficient way.

The source of this citation, as well as more information on that topic could be found in this blog post:

Should you still be using QGraphicsView?

Solution

If I must at any cost use the exact approach the OP uses, my solution would be to:

1. Use QGraphicsScene::selectionChanged to react to selection changes and
2. Use QGraphicsScene::selectedItems to check if an item is selected.

Example

Here is the MVCE the OP provided, modified by me to demonstrate how the proposed solution could be implemented:

#include <QApplication>
#include <QMainWindow>
#include <QGraphicsLinearLayout>
#include <QGraphicsProxyWidget>
#include <QGraphicsView>
#include <QPainter>

class SimpleItem : public QGraphicsItem
{
public :
    explicit SimpleItem(QGraphicsItem *parent = nullptr) :
        QGraphicsItem(parent) { setFlag(ItemIsSelectable, true); }

    QRectF boundingRect() const override { return QRectF(-20, -20, 40, 40); }
    void paint(QPainter *painter, const QStyleOptionGraphicsItem *,
               QWidget *) override {
        painter->setPen(isSelected() ? Qt::red : Qt::black);
        painter->setBrush(Qt::gray);
        painter->drawRect(boundingRect());
    }
};

class SceneItem : public QGraphicsWidget
{
    QGraphicsScene *m_scene;
public:
    explicit SceneItem(QGraphicsItem *parent = nullptr) :
        QGraphicsWidget(parent),
        m_scene(new QGraphicsScene(this)) {
        auto *layout = new QGraphicsLinearLayout;
        auto *view = new QGraphicsView(m_scene);
        auto *proxy = new QGraphicsProxyWidget(this);
        auto *simpleItem = new SimpleItem(this);

        m_scene->addItem(simpleItem);
        proxy->setWidget(view);
        layout->addItem(proxy);

        setLayout(layout);
        setFocusPolicy(Qt::ClickFocus);
        setFlag(QGraphicsItem::ItemIsSelectable, true);
    }

    QGraphicsScene *scene() const { return m_scene; }
    QRectF boundingRect() const override { return QRectF(0, 0, 100, 100);}
    void paint(QPainter *painter, const QStyleOptionGraphicsItem *,
               QWidget *) override {
        painter->setPen(isSelected() ? Qt::red : Qt::black);
        painter->setBrush(Qt::lightGray);
        painter->drawRect(boundingRect());
    }
    int type() const override { return UserType; }
};

int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    QMainWindow w;
    auto *scene = new QGraphicsScene;
    auto *view = new QGraphicsView(scene);
    auto *item = new SceneItem;

    scene->addItem(item);
    view->setDragMode(QGraphicsView::RubberBandDrag);

    QObject::connect(item->scene(), &QGraphicsScene::selectionChanged, [item, scene](){
        if (!item->scene()->selectedItems().isEmpty())
            scene->clearSelection();
    });

    QObject::connect(scene, &QGraphicsScene::selectionChanged, [scene](){
        if (scene->selectedItems().isEmpty())
            return;

        for (auto *item : scene->items())
            if (item->type() == QGraphicsItem::UserType)
                static_cast<SceneItem *>(item)->scene()->clearSelection();
    });

    w.setCentralWidget(view);
    w.show();

    return a.exec();
}